From 62924b6eaeb6e0cb69dcf1b42ee730c9094297e2 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 20 2017 15:35:36 +0000 Subject: [PATCH 1/57] Make no-debug a runserver option Signed-off-by: Patrick Uiterwijk --- diff --git a/runserver.py b/runserver.py index b774b75..2fd8daf 100755 --- a/runserver.py +++ b/runserver.py @@ -30,6 +30,9 @@ parser.add_argument( '--port', '-p', default=5000, help='Port for the Pagure to run on.') parser.add_argument( + '--no-debug', action='store_true', + help='Disable debugging') +parser.add_argument( '--host', default="127.0.0.1", help='Hostname to listen on. When set to 0.0.0.0 the server is available ' 'externally. Defaults to 127.0.0.1 making the it only visible on localhost') @@ -54,5 +57,5 @@ if args.profile: APP.config['PROFILE'] = True APP.wsgi_app = ProfilerMiddleware(APP.wsgi_app, restrictions=[30]) -APP.debug = True +APP.debug = not args.no_debug APP.run(host=args.host, port=int(args.port)) From 409b0c6734f3d1871d4c743ae00fc269a27bd4df Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:58 +0000 Subject: [PATCH 2/57] Docker-compose frontend Signed-off-by: Patrick Uiterwijk --- diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ad93b8b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3.2' +volumes: + repos: +services: + web: + build: + context: ./docker + dockerfile: web + depends_on: + - redis + - postgresql + image: pagure-web:latest + ports: + - "5000:5000" + volumes: + - type: volume + source: repos + target: /repos + read_only: true + - .:/code + redis: + image: redis + postgresql: + image: postgres + environment: + - POSTGRES_USER=pagure + - POSTGRES_PASSWORD=pagure + - POSTGRES_DB=pagure + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - ./docker/volumes/postgres:/var/lib/postgresql/data/pgdata diff --git a/docker/web b/docker/web new file mode 100644 index 0000000..636bcfd --- /dev/null +++ b/docker/web @@ -0,0 +1,22 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 + +WORKDIR /code +ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg"] +EXPOSE 5000 + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code diff --git a/openshift.cfg b/openshift.cfg new file mode 100644 index 0000000..8cdd73b --- /dev/null +++ b/openshift.cfg @@ -0,0 +1,11 @@ +SECRET_KEY = 'klalkdsaskrhjklh3423423' +DB_URL = 'postgresql://pagure:pagure@postgresql/pagure' +INSTANCE_NAME = 'DEVELOPMENT PAGURE' +EMAIL_ERROR = '' +APP_URL = 'https://web-pagure.b6ff.rh-idev.openshiftapps.com/' +REDIS_HOST = 'redis' +GIT_FOLDER = '/repos/repos' +DOCS_FOLDER = '/repos/docs' +TICKETS_FOLDER = '/repos/tickets' +REQUESTS_FOLDER = '/repos/requests' +REMOTE_GIT_FOLDER = '/repos/remote' From 57e33fdf96ccb99268cbda1560c725991bc2b6d3 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:58 +0000 Subject: [PATCH 3/57] pagure_git docker Signed-off-by: Patrick Uiterwijk --- diff --git a/docker-compose.yml b/docker-compose.yml index ad93b8b..de3bbb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,23 @@ services: target: /repos read_only: true - .:/code + backend_git: + build: + context: ./docker + dockerfile: backend-git + depends_on: + - redis + - postgresql + image: pagure-backend-git:latest + volumes: + - type: volume + source: repos + target: /repos + read_only: false + - .:/code + environment: + - PYTHONPATH=. + - PAGURE_CONFIG=/code/openshift.cfg redis: image: redis postgresql: diff --git a/docker/backend-git b/docker/backend-git new file mode 100644 index 0000000..ee533f5 --- /dev/null +++ b/docker/backend-git @@ -0,0 +1,23 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 + +RUN dnf install -y python-trollius python-trollius-redis + +WORKDIR /code +ENTRYPOINT ["/usr/bin/python", "/code/pagure_git/pagure_git_server.py"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code From b84a966f9d36b9b9ed04889f8dfdab204a2fe66a Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:58 +0000 Subject: [PATCH 4/57] Make webhook server use redis host config Signed-off-by: Patrick Uiterwijk --- diff --git a/webhook-server/pagure-webhook-server.py b/webhook-server/pagure-webhook-server.py index 4644867..cbcb1dc 100644 --- a/webhook-server/pagure-webhook-server.py +++ b/webhook-server/pagure-webhook-server.py @@ -105,50 +105,53 @@ def call_web_hooks(project, topic, msg, urls): @trollius.coroutine def handle_messages(): - connection = yield trollius.From(trollius_redis.Connection.create( - host='0.0.0.0', port=6379, db=0)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.hook'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - username = None - if data['project'].startswith('forks'): - username, projectname = data['project'].split('/', 2)[1:] - else: - projectname = data['project'] - - namespace = None - if '/' in projectname: - namespace, projectname = projectname.split('/', 1) - - log.info( - 'Searching %s/%s/%s' % (username, namespace, projectname)) - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - project = pagure.lib._get_project( - session=session, name=projectname, user=username, - namespace=namespace) - if not project: - log.info('No project found with these criteria') - session.close() - continue - urls = project.settings.get('Web-hooks') + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.hook'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + username = None + if data['project'].startswith('forks'): + username, projectname = data['project'].split('/', 2)[1:] + else: + projectname = data['project'] + + namespace = None + if '/' in projectname: + namespace, projectname = projectname.split('/', 1) + + log.info( + 'Searching %s/%s/%s' % (username, namespace, projectname)) + session = pagure.lib.create_session(pagure.APP.config['DB_URL']) + project = pagure.lib._get_project( + session=session, name=projectname, user=username, + namespace=namespace) + if not project: + log.info('No project found with these criteria') session.close() - if not urls: - log.info('No URLs set: %s' % urls) - continue - urls = urls.split('\n') - log.info('Got the project, going to the webhooks') - call_web_hooks(project, data['topic'], data['msg'], urls) + continue + urls = project.settings.get('Web-hooks') + session.close() + if not urls: + log.info('No URLs set: %s' % urls) + continue + urls = urls.split('\n') + log.info('Got the project, going to the webhooks') + call_web_hooks(project, data['topic'], data['msg'], urls) def main(): From 6cfcf7ae59e64e73fcad83d14a654ff4145c6492 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:58 +0000 Subject: [PATCH 5/57] Fix docker-web Signed-off-by: Patrick Uiterwijk --- diff --git a/docker/web b/docker/web index 636bcfd..ac0eb62 100644 --- a/docker/web +++ b/docker/web @@ -14,7 +14,7 @@ RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python- python-fedora-flask python2-pillow python2-psycopg2 WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg"] +ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg", "--no-debug"] EXPOSE 5000 # Code injection is last to make optimal use of caches From fc4636d798ccf28e33412670c412ddd5053d4ba0 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 6/57] Pagure_git skeleton Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure_git/pagure_git.service b/pagure_git/pagure_git.service new file mode 100644 index 0000000..66a6563 --- /dev/null +++ b/pagure_git/pagure_git.service @@ -0,0 +1,14 @@ +[Unit] +Description=Pagure GIT Server +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=/usr/libexec/pagure-git/pagure_git_server.py +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/pagure_git/pagure_git_server.py b/pagure_git/pagure_git_server.py new file mode 100644 index 0000000..1334c69 --- /dev/null +++ b/pagure_git/pagure_git_server.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +""" + (c) 2017 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + + +GIT server for Pagure's editing features. +This server takes messages sent to the redis queue and handles git change +operations. + +This service is required for the core Pagure functionality, since it implements +the creation of new repos, forking of repos and creating and merging pull +requests. +""" + +import logging +import os +import urlparse + +import trollius +import trollius_redis + +log = logging.getLogger(__name__) + + +if 'PAGURE_CONFIG' not in os.environ \ + and os.path.exists('/etc/pagure/pagure.cfg'): + print 'Using configuration file `/etc/pagure/pagure.cfg`' + os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' + + +import pagure # noqa: E402 +import pagure.lib # noqa: E402 +from pagure.exceptions import PagureEvException # noqa: E402 + +SERVER = None + + +@trollius.coroutine +def handle_messages(): + host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') + port = pagure.APP.config.get('REDIS_PORT', 6379) + dbname = pagure.APP.config.get('REDIS_DB', 0) + connection = yield trollius.From(trollius_redis.Connection.create( + host=host, port=port, db=dbname)) + + # Create subscriber. + subscriber = yield trollius.From(connection.start_subscribe()) + + # Subscribe to channel. + yield trollius.From(subscriber.subscribe(['pagure.git'])) + + # Inside a while loop, wait for incoming events. + while True: + reply = yield trollius.From(subscriber.next_published()) + log.info( + 'Received: %s on channel: %s', + repr(reply.value), reply.channel) + data = json.loads(reply.value) + + +def main(): + server = None + try: + loop = trollius.get_event_loop() + tasks = [ + trollius.async(handle_messages()), + ] + loop.run_until_complete(trollius.wait(tasks)) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError: + pass + + log.info("End Connection") + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() From 0f077b9b81c9147216a34e835ffa6d452cff1e52 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 7/57] WIP ALL THE THINGS Signed-off-by: Patrick Uiterwijk --- diff --git a/docker-compose.yml b/docker-compose.yml index de3bbb9..aa5ee9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,14 +18,14 @@ services: target: /repos read_only: true - .:/code - backend_git: + worker: build: context: ./docker - dockerfile: backend-git + dockerfile: worker depends_on: - redis - postgresql - image: pagure-backend-git:latest + image: pagure-worker:latest volumes: - type: volume source: repos diff --git a/docker/backend-git b/docker/backend-git deleted file mode 100644 index ee533f5..0000000 --- a/docker/backend-git +++ /dev/null @@ -1,23 +0,0 @@ -FROM registry.fedoraproject.org/fedora:25 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-flask-multistatic python-markdown python-psutil \ - python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 - -RUN dnf install -y python-trollius python-trollius-redis - -WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/pagure_git/pagure_git_server.py"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code diff --git a/docker/web b/docker/web index ac0eb62..0d434e1 100644 --- a/docker/web +++ b/docker/web @@ -13,8 +13,11 @@ RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python- python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ python-fedora-flask python2-pillow python2-psycopg2 +RUN dnf install -y python2-celery + WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg", "--no-debug"] +# Openshift: --no-debug +ENTRYPOINT ["/usr/bin/python", "/code/runserver.py", "--host", "0.0.0.0", "--config", "/code/openshift.cfg"] EXPOSE 5000 # Code injection is last to make optimal use of caches diff --git a/docker/worker b/docker/worker new file mode 100644 index 0000000..5e592f7 --- /dev/null +++ b/docker/worker @@ -0,0 +1,28 @@ +FROM registry.fedoraproject.org/fedora:25 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-flask-multistatic python-markdown python-psutil \ + python-pygit2 python-pygments python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 + +RUN dnf install -y python2-celery + +RUN useradd docker && \ + chown docker:docker /repos + +WORKDIR /code +ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info", "--autoreload"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code + +USER docker diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index e978f06..e85f184 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -53,6 +53,7 @@ import pagure.lib.notify import pagure.lib.plugins import pagure.pfmarkdown from pagure.lib import model +from pagure.lib import tasks REDIS = None @@ -1270,6 +1271,8 @@ def new_project(session, user, name, blacklist, allowed_prefix, prevent_40_chars=False, namespace=None, user_ns=False, ignore_existing_repo=False, private=False): ''' Create a new project based on the information provided. + + Is an async operation, and returns task ID. ''' if (not namespace and name in blacklist) \ or (namespace and '%s/%s' % (namespace, name) in blacklist): @@ -1342,6 +1345,21 @@ def new_project(session, user, name, blacklist, allowed_prefix, # Make sure we won't have SQLAlchemy error before we create the repo session.flush() + # Register creation et al + log_action(session, 'created', project, user_obj) + + pagure.lib.notify.log( + project, + topic='project.new', + msg=dict( + project=project.to_json(public=True), + agent=user_obj.username, + ), + ) + + return tasks.create_project.delay(namespace, name, add_readme, + ignore_existing_repo).id + # Add the readme file if it was asked if not add_readme: pygit2.init_repository(gitrepo, bare=True) @@ -1419,24 +1437,6 @@ def new_project(session, user, name, blacklist, allowed_prefix, plugin.set_up(project) plugin.install(project, dbobj) - # create the project in the db - session.commit() - - log_action(session, 'created', project, user_obj) - - pagure.lib.notify.log( - project, - topic='project.new', - msg=dict( - project=project.to_json(public=True), - agent=user_obj.username, - ), - ) - - return 'Project "%s" created' % ( - '%s/%s' % (project.namespace, project.name) if project.namespace - else project.name) - def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, issue_uid=None, private=False, status=None, close_status=None, diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py new file mode 100644 index 0000000..eb7b2b6 --- /dev/null +++ b/pagure/lib/tasks.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2017 - Copyright Red Hat Inc + + Authors: + Patrick Uiterwijk + +""" + +from celery import Celery +from celery.result import AsyncResult + +import pagure +import pagure.lib + + +conn = Celery('tasks', + broker='redis://%s' % pagure.APP.config['REDIS_HOST'], + backend='redis://%s' % pagure.APP.config['REDIS_HOST']) + + +def get_result(uuid): + return AsyncResult(uuid, conn.backend) + + +def ret(endpoint, **kwargs): + toret = {'endpoint': endpoint} + toret.update(kwargs) + return toret + + +@conn.task +def generate_gitolite_acls(): + # TODO: Implement gitolite acl stuff + return 'TODO' + + +@conn.task +def create_project(namespace, name, add_readme, ignore_existing_repo): + # TODO: Implement creation (see pagure.lib.new_project after return) + generate_gitolite_acls.delay() + return ret('view_repo', repo=name, namespace=namespace) diff --git a/pagure/templates/waiting.html b/pagure/templates/waiting.html new file mode 100644 index 0000000..6e4ff18 --- /dev/null +++ b/pagure/templates/waiting.html @@ -0,0 +1,20 @@ +{% extends "master.html" %} + +{% block title %}Waiting{% endblock %} + +{% block header %} + +{% endblock %} + +{% block content %} + +

+ Waiting +

+ +

+ We are waiting for your task to finish. + This page should be refreshed automatically, but if not click Here +

+ +{% endblock %} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index ed8e78c..19f7d66 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -452,7 +452,7 @@ def new_project(): namespace = namespace.strip() try: - pagure.lib.new_project( + taskid = pagure.lib.new_project( SESSION, name=name, private=private, @@ -474,9 +474,8 @@ def new_project(): user_ns=APP.config.get('USER_NAMESPACE', False), ) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() return flask.redirect(flask.url_for( - 'view_repo', repo=name, namespace=namespace)) + 'wait_task', taskid=taskid)) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover @@ -489,6 +488,20 @@ def new_project(): ) +@APP.route('/wait/') +def wait_task(taskid): + result = pagure.lib.tasks.get_result(taskid) + if result.ready: + result = result.get(timeout=0) + print result + endpoint = result.pop('endpoint') + return flask.redirect( + flask.url_for(endpoint, **result)) + else: + return flask.render_template( + 'waiting.html', + taskid=taskid) + @APP.route('/settings/', methods=('GET', 'POST')) @APP.route('/settings', methods=('GET', 'POST')) @login_required diff --git a/pagure_git/pagure_git.service b/pagure_git/pagure_git.service deleted file mode 100644 index 66a6563..0000000 --- a/pagure_git/pagure_git.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Pagure GIT Server -After=redis.target -Documentation=https://pagure.io/pagure - -[Service] -ExecStart=/usr/libexec/pagure-git/pagure_git_server.py -Type=simple -User=git -Group=git -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/pagure_git/pagure_git_server.py b/pagure_git/pagure_git_server.py deleted file mode 100644 index 1334c69..0000000 --- a/pagure_git/pagure_git_server.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" - (c) 2017 - Copyright Red Hat Inc - - Authors: - Patrick Uiterwijk - - -GIT server for Pagure's editing features. -This server takes messages sent to the redis queue and handles git change -operations. - -This service is required for the core Pagure functionality, since it implements -the creation of new repos, forking of repos and creating and merging pull -requests. -""" - -import logging -import os -import urlparse - -import trollius -import trollius_redis - -log = logging.getLogger(__name__) - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - - -import pagure # noqa: E402 -import pagure.lib # noqa: E402 -from pagure.exceptions import PagureEvException # noqa: E402 - -SERVER = None - - -@trollius.coroutine -def handle_messages(): - host = pagure.APP.config.get('REDIS_HOST', '0.0.0.0') - port = pagure.APP.config.get('REDIS_PORT', 6379) - dbname = pagure.APP.config.get('REDIS_DB', 0) - connection = yield trollius.From(trollius_redis.Connection.create( - host=host, port=port, db=dbname)) - - # Create subscriber. - subscriber = yield trollius.From(connection.start_subscribe()) - - # Subscribe to channel. - yield trollius.From(subscriber.subscribe(['pagure.git'])) - - # Inside a while loop, wait for incoming events. - while True: - reply = yield trollius.From(subscriber.next_published()) - log.info( - 'Received: %s on channel: %s', - repr(reply.value), reply.channel) - data = json.loads(reply.value) - - -def main(): - server = None - try: - loop = trollius.get_event_loop() - tasks = [ - trollius.async(handle_messages()), - ] - loop.run_until_complete(trollius.wait(tasks)) - loop.run_forever() - except KeyboardInterrupt: - pass - except trollius.ConnectionResetError: - pass - - log.info("End Connection") - loop.close() - log.info("End") - - -if __name__ == '__main__': - log = logging.getLogger("") - formatter = logging.Formatter( - "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") - - # setup console logging - log.setLevel(logging.DEBUG) - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - aslog = logging.getLogger("asyncio") - aslog.setLevel(logging.DEBUG) - - ch.setFormatter(formatter) - log.addHandler(ch) - main() From c953939d1a13332c772eb59b2962d6188e8a4b03 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 8/57] Remove locking in web UI Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 2c7fc2d..98f2705 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -401,29 +401,6 @@ def generate_user_key_files(): pagure.lib.git.generate_gitolite_acls() -def acquire_lock(function): - """ Flask decorator to indicate the repo needs to be locked. - - This function reretrieves the flask.g.repo object, but this time requests - that the repo object gets locked. - This lock is retrieved in a way that actively waits until the lock is - acquired. - """ - @wraps(function) - def decorated_function(*args, **kwargs): - set_variables(with_lock=True) - return function(*args, **kwargs) - return decorated_function - - -def ensure_lock(repo): - """ Function to make sure that `repo` was retrieved locked. """ - if not flask.g.repo_locked: - raise Exception('Repo was not locked') - if repo is not flask.g.repo: - raise Exception('Incorrect repo was locked') - - def login_required(function): """ Flask decorator to retrict access to logged in user. If the auth system is ``fas`` it will also require that the user sign diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index b4c6ecb..31dc14a 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -30,8 +30,7 @@ import pagure.exceptions import pagure.lib import pagure.lib.git import pagure.forms -from pagure import (APP, SESSION, login_required, __get_file_in_tree, - acquire_lock) +from pagure import (APP, SESSION, login_required, __get_file_in_tree) _log = logging.getLogger(__name__) @@ -171,7 +170,6 @@ def request_pulls(repo, username=None, namespace=None): '/fork////pull-request//') @APP.route( '/fork////pull-request/') -@acquire_lock def request_pull(repo, requestid, username=None, namespace=None): """ Create a pull request with the changes from the fork into the project. """ @@ -261,7 +259,6 @@ def request_pull(repo, requestid, username=None, namespace=None): @APP.route('/fork///pull-request/.patch') @APP.route( '/fork////pull-request/.patch') -@acquire_lock def request_pull_patch(repo, requestid, username=None, namespace=None): """ Returns the commits from the specified pull-request as patches. """ @@ -351,7 +348,6 @@ def request_pull_patch(repo, requestid, username=None, namespace=None): '/fork////pull-request//edit', methods=('GET', 'POST')) @login_required -@acquire_lock def request_pull_edit(repo, requestid, username=None, namespace=None): """ Edit the title of a pull-request. """ @@ -425,7 +421,6 @@ def request_pull_edit(repo, requestid, username=None, namespace=None): '/fork////pull-request//' 'comment///', methods=('GET', 'POST')) @login_required -@acquire_lock def pull_request_add_comment( repo, requestid, commit=None, filename=None, row=None, username=None, namespace=None): @@ -513,7 +508,6 @@ def pull_request_add_comment( '/fork////pull-request//' 'comment/drop', methods=['POST']) @login_required -@acquire_lock def pull_request_drop_comment( repo, requestid, username=None, namespace=None): """ Delete a comment of a pull-request. @@ -587,7 +581,6 @@ def pull_request_drop_comment( '/comment//edit', methods=('GET', 'POST')) @login_required -@acquire_lock def pull_request_edit_comment( repo, requestid, commentid, username=None, namespace=None): """Edit comment of a pull request @@ -674,7 +667,6 @@ def pull_request_edit_comment( '/fork////pull-request//merge', methods=['POST']) @login_required -@acquire_lock def merge_request_pull(repo, requestid, username=None, namespace=None): """ Create a pull request with the changes from the fork into the project. """ @@ -764,7 +756,6 @@ def merge_request_pull(repo, requestid, username=None, namespace=None): '/fork////pull-request/cancel/', methods=['POST']) @login_required -@acquire_lock def cancel_request_pull(repo, requestid, username=None, namespace=None): """ Cancel a pull request. """ @@ -820,7 +811,6 @@ def cancel_request_pull(repo, requestid, username=None, namespace=None): '/fork////pull-request//assign', methods=['POST']) @login_required -@acquire_lock def set_assignee_requests(repo, requestid, username=None, namespace=None): ''' Assign a pull-request. ''' repo = flask.g.repo @@ -941,7 +931,6 @@ def fork_project(repo, username=None, namespace=None): @APP.route( '/fork////diff/' '..', methods=('GET', 'POST')) -@acquire_lock def new_request_pull( repo, branch_to, branch_from, username=None, namespace=None): """ Create a pull request with the changes from the fork into the project. @@ -1088,7 +1077,6 @@ def new_request_pull( '/fork////diff/remote', methods=('GET', 'POST')) @login_required -@acquire_lock def new_remote_request_pull(repo, username=None, namespace=None): """ Create a pull request with the changes from a remote fork into the project. @@ -1234,7 +1222,6 @@ def new_remote_request_pull(repo, username=None, namespace=None): '/fork_edit/fork////edit//' 'f/', methods=['POST']) @login_required -@acquire_lock def fork_edit_file( repo, branchname, filename, username=None, namespace=None): """ Fork the project specified and open the specific file to edit diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 9cb2290..7d1687c 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -36,8 +36,7 @@ import pagure.lib import pagure.lib.encoding_utils import pagure.forms from pagure import (APP, SESSION, __get_file_in_tree, - login_required, authenticated, urlpattern, - acquire_lock) + login_required, authenticated, urlpattern) _log = logging.getLogger(__name__) @@ -70,7 +69,6 @@ _log = logging.getLogger(__name__) '/fork////issue//update', methods=['GET', 'POST']) @login_required -@acquire_lock def update_issue(repo, issueid, username=None, namespace=None): ''' Add a comment to an issue. ''' is_js = flask.request.args.get('js', False) @@ -379,7 +377,6 @@ def update_issue(repo, issueid, username=None, namespace=None): '/fork////tag//edit', methods=('GET', 'POST')) @login_required -@acquire_lock def edit_tag(repo, tag, username=None, namespace=None): """ Edit the specified tag associated with the issues of a project. """ @@ -449,7 +446,6 @@ def edit_tag(repo, tag, username=None, namespace=None): @APP.route('//update/tags', methods=['POST']) @APP.route('///update/tags', methods=['POST']) @login_required -@acquire_lock def update_tags(repo, username=None, namespace=None): """ Update the tags of a project. """ @@ -543,7 +539,6 @@ def update_tags(repo, username=None, namespace=None): @APP.route('/fork///droptag/', methods=['POST']) @APP.route('/fork////droptag/', methods=['POST']) @login_required -@acquire_lock def remove_tag(repo, username=None, namespace=None): """ Remove the specified tag, associated with the issues, from the project. """ @@ -872,7 +867,6 @@ def view_roadmap(repo, username=None, namespace=None): '/fork////new_issue', methods=('GET', 'POST')) @login_required -@acquire_lock def new_issue(repo, username=None, namespace=None): """ Create a new issue """ @@ -1053,7 +1047,6 @@ def view_issue(repo, issueid, username=None, namespace=None): methods=['POST']) @APP.route('/fork////issue//drop', methods=['POST']) -@acquire_lock def delete_issue(repo, issueid, username=None, namespace=None): """ Delete the specified issue """ @@ -1113,7 +1106,6 @@ def delete_issue(repo, issueid, username=None, namespace=None): @APP.route('/fork////issue//edit', methods=('GET', 'POST')) @login_required -@acquire_lock def edit_issue(repo, issueid, username=None, namespace=None): """ Edit the specified issue """ @@ -1233,7 +1225,6 @@ def edit_issue(repo, issueid, username=None, namespace=None): @APP.route('/fork////issue//upload', methods=['POST']) @login_required -@acquire_lock def upload_issue(repo, issueid, username=None, namespace=None): ''' Upload a file to a ticket. ''' @@ -1376,7 +1367,6 @@ def view_issue_raw_file( @APP.route('/fork////issue//comment' '//edit', methods=('GET', 'POST')) @login_required -@acquire_lock def edit_comment_issue( repo, issueid, commentid, username=None, namespace=None): """Edit comment of an issue @@ -1457,7 +1447,6 @@ def edit_comment_issue( @APP.route( '/fork////issues/reports', methods=['POST']) @login_required -@acquire_lock def save_reports(repo, username=None, namespace=None): """ Marked for watching or Unwatching """ diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 1915037..5e4acd5 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -51,7 +51,7 @@ import pagure.forms import pagure import pagure.ui.plugins from pagure import (APP, SESSION, __get_file_in_tree, login_required, - admin_session_timedout, acquire_lock) + admin_session_timedout) from pagure.lib import encoding_utils @@ -989,7 +989,6 @@ def new_release(repo, username=None, namespace=None): @APP.route( '/fork////settings', methods=('GET', 'POST')) @login_required -@acquire_lock def view_settings(repo, username=None, namespace=None): """ Presents the settings of the project. """ @@ -1353,7 +1352,6 @@ def update_milestones(repo, username=None, namespace=None): @APP.route( '/fork////default/branch/', methods=['POST']) @login_required -@acquire_lock def change_ref_head(repo, username=None, namespace=None): """ Change HEAD reference """ @@ -1397,7 +1395,6 @@ def change_ref_head(repo, username=None, namespace=None): @APP.route('/fork///delete', methods=['POST']) @APP.route('/fork////delete', methods=['POST']) @login_required -@acquire_lock def delete_repo(repo, username=None, namespace=None): """ Delete the present project. """ @@ -1906,7 +1903,6 @@ def add_group_project(repo, username=None, namespace=None): @APP.route('/fork///regenerate', methods=['POST']) @APP.route('/fork////regenerate', methods=['POST']) @login_required -@acquire_lock def regenerate_git(repo, username=None, namespace=None): """ Regenerate the specified git repo with the content in the project. """ @@ -2099,7 +2095,6 @@ def revoke_api_token(repo, token_id, username=None, namespace=None): '/fork////edit//f/' '', methods=('GET', 'POST')) @login_required -@acquire_lock def edit_file(repo, branchname, filename, username=None, namespace=None): """ Edit a file online. """ @@ -2184,7 +2179,6 @@ def edit_file(repo, branchname, filename, username=None, namespace=None): @APP.route('/fork////b//delete', methods=['POST']) @login_required -@acquire_lock def delete_branch(repo, branchname, username=None, namespace=None): """ Delete the branch of a project. """ From c9d5be7c3fff5717221f85e9e0ff9a19d3597102 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 9/57] Worker docker Signed-off-by: Patrick Uiterwijk --- diff --git a/docker/worker b/docker/worker index 5e592f7..98cd896 100644 --- a/docker/worker +++ b/docker/worker @@ -15,14 +15,11 @@ RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python- RUN dnf install -y python2-celery -RUN useradd docker && \ - chown docker:docker /repos - WORKDIR /code ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info", "--autoreload"] # Code injection is last to make optimal use of caches VOLUME ["/code"] # Openshift: COPY / /code - -USER docker +# Ideally this would run as non-root, but that needs the /repos owned correctly +ENV C_FORCE_ROOT true From 4e3dbd3ab1c94be10f766fe399bf4636833b6a6b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 10/57] Move generate_gitolote_acl to task Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/cli/admin.py b/pagure/cli/admin.py index cc840ca..0d1482d 100644 --- a/pagure/cli/admin.py +++ b/pagure/cli/admin.py @@ -172,7 +172,7 @@ def do_generate_acl(_): 'Do you want to re-generate the gitolite.conf file then ' 'calling: %s' % cmd) if _ask_confirmation(): - pagure.lib.git.generate_gitolite_acls() + pagure.lib.git._generate_gitolite_acls() print('Gitolite ACLs updated') diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 675ab7f..94973a1 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -34,6 +34,7 @@ import pagure.lib import pagure.lib.notify from pagure.lib import model from pagure.lib.repo import PagureRepo +from pagure.lib import tasks _log = logging.getLogger(__name__) @@ -183,6 +184,10 @@ def _get_gitolite_command(): def generate_gitolite_acls(): + tasks.generate_gitolite_acls.delay() + + +def _generate_gitolite_acls(): """ Generate the gitolite configuration file for all repos """ _log.info('Refresh gitolite configuration') diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index eb7b2b6..151e744 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -8,16 +8,26 @@ """ +import os +import os.path +import shutil + from celery import Celery from celery.result import AsyncResult +import pygit2 +import tempfile +import six + import pagure +from pagure import APP import pagure.lib +import pagure.lib.git conn = Celery('tasks', - broker='redis://%s' % pagure.APP.config['REDIS_HOST'], - backend='redis://%s' % pagure.APP.config['REDIS_HOST']) + broker='redis://%s' % APP.config['REDIS_HOST'], + backend='redis://%s' % APP.config['REDIS_HOST']) def get_result(uuid): @@ -32,12 +42,94 @@ def ret(endpoint, **kwargs): @conn.task def generate_gitolite_acls(): - # TODO: Implement gitolite acl stuff - return 'TODO' + pagure.lib.git._generate_gitolite_acls() @conn.task -def create_project(namespace, name, add_readme, ignore_existing_repo): - # TODO: Implement creation (see pagure.lib.new_project after return) +def create_project(username, namespace, name, add_readme, ignore_existing_repo): + project = pagure.lib._get_project(pagure.SESSION, namespace=namespace, + name=name, with_lock=True) + userobj = pagure.lib.search_user(pagure.SESSION, username=username) + gitrepo = os.path.join(APP.config['GIT_FOLDER'], project.path) + + # Add the readme file if it was asked + if not add_readme: + pygit2.init_repository(gitrepo, bare=True) + else: + temp_gitrepo_path = tempfile.mkdtemp(prefix='pagure-') + temp_gitrepo = pygit2.init_repository(temp_gitrepo_path, bare=False) + author = userobj.fullname or userobj.user + author_email = userobj.default_email + if six.PY2: + author = author.encode('utf-8') + author_email = author_email.encode('utf-8') + author = pygit2.Signature(author, author_email) + content = u"# %s\n\n%s" % (name, project.description) + readme_file = os.path.join(temp_gitrepo.workdir, "README.md") + with open(readme_file, 'wb') as stream: + stream.write(content.encode('utf-8')) + temp_gitrepo.index.add_all() + temp_gitrepo.index.write() + tree = temp_gitrepo.index.write_tree() + temp_gitrepo.create_commit( + 'HEAD', author, author, 'Added the README', tree, []) + pygit2.clone_repository(temp_gitrepo_path, gitrepo, bare=True) + shutil.rmtree(temp_gitrepo_path) + + # Make the repo exportable via apache + http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w') as stream: + pass + + docrepo = os.path.join(APP.config['DOCS_FOLDER'], project.path) + if os.path.exists(docrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + raise pagure.exceptions.RepoExistsException( + 'The docs repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository(docrepo, bare=True) + + ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], project.path) + if os.path.exists(ticketrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + shutil.rmtree(docrepo) + raise pagure.exceptions.RepoExistsException( + 'The tickets repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository( + ticketrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], project.path) + if os.path.exists(requestrepo): + if not ignore_existing_repo: + shutil.rmtree(gitrepo) + shutil.rmtree(docrepo) + shutil.rmtree(ticketrepo) + raise pagure.exceptions.RepoExistsException( + 'The requests repo "%s" already exists' % project.path + ) + else: + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + # Install the default hook + # TODO: This needs enabling, but doesn't work in my env + #plugin = pagure.lib.plugins.get_plugin('default') + #dbobj = plugin.db_object() + #dbobj.active = True + #dbobj.project_id = project.id + #pagure.SESSION.add(dbobj) + #pagure.SESSION.flush() + #plugin.set_up(project) + #plugin.install(project, dbobj) + #pagure.SESSION.commit() + generate_gitolite_acls.delay() return ret('view_repo', repo=name, namespace=namespace) From 56ff97f846eacb02522966ee0936962c7f1c7f39 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 11/57] Make new repo creation part of tasks Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index e85f184..c997b60 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1357,85 +1357,8 @@ def new_project(session, user, name, blacklist, allowed_prefix, ), ) - return tasks.create_project.delay(namespace, name, add_readme, - ignore_existing_repo).id - - # Add the readme file if it was asked - if not add_readme: - pygit2.init_repository(gitrepo, bare=True) - else: - temp_gitrepo_path = tempfile.mkdtemp(prefix='pagure-') - temp_gitrepo = pygit2.init_repository(temp_gitrepo_path, bare=False) - author = userobj.fullname or userobj.user - author_email = userobj.default_email - if six.PY2: - author = author.encode('utf-8') - author_email = author_email.encode('utf-8') - author = pygit2.Signature(author, author_email) - content = u"# %s\n\n%s" % (name, description) - readme_file = os.path.join(temp_gitrepo.workdir, "README.md") - with open(readme_file, 'wb') as stream: - stream.write(content.encode('utf-8')) - temp_gitrepo.index.add_all() - temp_gitrepo.index.write() - tree = temp_gitrepo.index.write_tree() - temp_gitrepo.create_commit( - 'HEAD', author, author, 'Added the README', tree, []) - pygit2.clone_repository(temp_gitrepo_path, gitrepo, bare=True) - shutil.rmtree(temp_gitrepo_path) - - # Make the repo exportable via apache - http_clone_file = os.path.join(gitrepo, 'git-daemon-export-ok') - if not os.path.exists(http_clone_file): - with open(http_clone_file, 'w') as stream: - pass - - docrepo = os.path.join(docfolder, project.path) - if os.path.exists(docrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - raise pagure.exceptions.RepoExistsException( - 'The docs repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository(docrepo, bare=True) - - ticketrepo = os.path.join(ticketfolder, project.path) - if os.path.exists(ticketrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - shutil.rmtree(docrepo) - raise pagure.exceptions.RepoExistsException( - 'The tickets repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository( - ticketrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - requestrepo = os.path.join(requestfolder, project.path) - if os.path.exists(requestrepo): - if not ignore_existing_repo: - shutil.rmtree(gitrepo) - shutil.rmtree(docrepo) - shutil.rmtree(ticketrepo) - raise pagure.exceptions.RepoExistsException( - 'The requests repo "%s" already exists' % project.path - ) - else: - pygit2.init_repository( - requestrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - # Install the default hook - plugin = pagure.lib.plugins.get_plugin('default') - dbobj = plugin.db_object() - dbobj.active = True - dbobj.project_id = project.id - session.add(dbobj) - session.flush() - plugin.set_up(project) - plugin.install(project, dbobj) + return tasks.create_project.delay(user_obj.username, namespace, name, + add_readme, ignore_existing_repo).id def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, From b4c473636dd560ea044160eeef213ccc3f03c52b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 12/57] No more locking Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/test_pagure_flask_dump_load_ticket.py b/tests/test_pagure_flask_dump_load_ticket.py index f2da049..48ff558 100644 --- a/tests/test_pagure_flask_dump_load_ticket.py +++ b/tests/test_pagure_flask_dump_load_ticket.py @@ -55,9 +55,8 @@ class PagureFlaskDumpLoadTicketTests(tests.Modeltests): self.path, 'requests') self.app = pagure.APP.test_client() - @patch('pagure.ensure_lock') @patch('pagure.lib.notify.send_email') - def test_dumping_reloading_ticket(self, elock, send_email): + def test_dumping_reloading_ticket(self, send_email): """ Test dumping a ticket into a JSON blob. """ send_email.return_value = True diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index 5511c1e..34d14ee 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -2598,88 +2598,87 @@ index 0000000..fb7093d 'Forks 0', output.data) - with patch('pagure.ensure_lock'): - # add issues - repo = pagure.get_authorized_project(self.session, 'test') - msg = pagure.lib.new_issue( - session=self.session, - repo=repo, - title='Test issue', - content='We should work on this', - user='pingou', - ticketfolder=os.path.join(self.path, 'tickets') - ) - self.session.commit() - self.assertEqual(msg.title, 'Test issue') - - msg = pagure.lib.new_issue( - session=self.session, - repo=repo, - title='Test issue #2', - content='We should work on this, really', - user='pingou', - ticketfolder=os.path.join(self.path, 'tickets') - ) - self.session.commit() - self.assertEqual(msg.title, 'Test issue #2') - - # Add a comment to an issue - issue = pagure.lib.search_issues(self.session, repo, issueid=1) - msg = pagure.lib.add_issue_comment( - session=self.session, - issue=issue, - comment='Hey look a comment!', - user='foo', - ticketfolder=None - ) - self.session.commit() - self.assertEqual(msg, 'Comment added') - - # add pull-requests - req = pagure.lib.new_pull_request( - session=self.session, - repo_from=repo, - branch_from='feature', - repo_to=repo, - branch_to='master', - title='test pull-request', - user='pingou', - requestfolder=os.path.join(self.path, 'requests'), - ) - self.session.commit() - self.assertEqual(req.id, 3) - self.assertEqual(req.title, 'test pull-request') - - req = pagure.lib.new_pull_request( - session=self.session, - repo_from=repo, - branch_from='feature2', - repo_to=repo, - branch_to='master', - title='test pull-request', - user='pingou', - requestfolder=os.path.join(self.path, 'requests'), - ) - self.session.commit() - self.assertEqual(req.id, 4) - self.assertEqual(req.title, 'test pull-request') - - # Add comment on a pull-request - request = pagure.lib.search_pull_requests( - self.session, requestid=3) - - msg = pagure.lib.add_pull_request_comment( - session=self.session, - request=request, - commit='commithash', - tree_id=None, - filename='file', - row=None, - comment='This is awesome, I got to remember it!', - user='foo', - requestfolder=None, - ) - self.assertEqual(msg, 'Comment added') + # add issues + repo = pagure.get_authorized_project(self.session, 'test') + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title='Test issue', + content='We should work on this', + user='pingou', + ticketfolder=os.path.join(self.path, 'tickets') + ) + self.session.commit() + self.assertEqual(msg.title, 'Test issue') + + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title='Test issue #2', + content='We should work on this, really', + user='pingou', + ticketfolder=os.path.join(self.path, 'tickets') + ) + self.session.commit() + self.assertEqual(msg.title, 'Test issue #2') + + # Add a comment to an issue + issue = pagure.lib.search_issues(self.session, repo, issueid=1) + msg = pagure.lib.add_issue_comment( + session=self.session, + issue=issue, + comment='Hey look a comment!', + user='foo', + ticketfolder=None + ) + self.session.commit() + self.assertEqual(msg, 'Comment added') + + # add pull-requests + req = pagure.lib.new_pull_request( + session=self.session, + repo_from=repo, + branch_from='feature', + repo_to=repo, + branch_to='master', + title='test pull-request', + user='pingou', + requestfolder=os.path.join(self.path, 'requests'), + ) + self.session.commit() + self.assertEqual(req.id, 3) + self.assertEqual(req.title, 'test pull-request') + + req = pagure.lib.new_pull_request( + session=self.session, + repo_from=repo, + branch_from='feature2', + repo_to=repo, + branch_to='master', + title='test pull-request', + user='pingou', + requestfolder=os.path.join(self.path, 'requests'), + ) + self.session.commit() + self.assertEqual(req.id, 4) + self.assertEqual(req.title, 'test pull-request') + + # Add comment on a pull-request + request = pagure.lib.search_pull_requests( + self.session, requestid=3) + + msg = pagure.lib.add_pull_request_comment( + session=self.session, + request=request, + commit='commithash', + tree_id=None, + filename='file', + row=None, + comment='This is awesome, I got to remember it!', + user='foo', + requestfolder=None, + ) + self.assertEqual(msg, 'Comment added') # Check before deleting the project output = self.app.get('/') @@ -2850,89 +2849,87 @@ index 0000000..fb7093d 'Forks 0', output.data) - # This part of the code calls lib stuff directly - with patch('pagure.ensure_lock'): - # add issues - repo = pagure.get_authorized_project(self.session, 'test') - msg = pagure.lib.new_issue( - session=self.session, - repo=repo, - title='Test issue', - content='We should work on this', - user='pingou', - ticketfolder=os.path.join(self.path, 'tickets') - ) - self.session.commit() - self.assertEqual(msg.title, 'Test issue') - - msg = pagure.lib.new_issue( - session=self.session, - repo=repo, - title='Test issue #2', - content='We should work on this, really', - user='pingou', - ticketfolder=os.path.join(self.path, 'tickets') - ) - self.session.commit() - self.assertEqual(msg.title, 'Test issue #2') - - # Add a comment to an issue - issue = pagure.lib.search_issues(self.session, repo, issueid=1) - msg = pagure.lib.add_issue_comment( - session=self.session, - issue=issue, - comment='Hey look a comment!', - user='foo', - ticketfolder=None - ) - self.session.commit() - self.assertEqual(msg, 'Comment added') - - # add pull-requests - req = pagure.lib.new_pull_request( - session=self.session, - repo_from=repo, - branch_from='feature', - repo_to=repo, - branch_to='master', - title='test pull-request', - user='pingou', - requestfolder=os.path.join(self.path, 'requests'), - ) - self.session.commit() - self.assertEqual(req.id, 3) - self.assertEqual(req.title, 'test pull-request') - - req = pagure.lib.new_pull_request( - session=self.session, - repo_from=repo, - branch_from='feature2', - repo_to=repo, - branch_to='master', - title='test pull-request', - user='pingou', - requestfolder=os.path.join(self.path, 'requests'), - ) - self.session.commit() - self.assertEqual(req.id, 4) - self.assertEqual(req.title, 'test pull-request') - - # Add comment on a pull-request - request = pagure.lib.search_pull_requests( - self.session, requestid=3) - - msg = pagure.lib.add_pull_request_comment( - session=self.session, - request=request, - commit='commithash', - tree_id=None, - filename='file', - row=None, - comment='This is awesome, I got to remember it!', - user='foo', - requestfolder=None, - ) - self.assertEqual(msg, 'Comment added') + # add issues + repo = pagure.get_authorized_project(self.session, 'test') + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title='Test issue', + content='We should work on this', + user='pingou', + ticketfolder=os.path.join(self.path, 'tickets') + ) + self.session.commit() + self.assertEqual(msg.title, 'Test issue') + + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title='Test issue #2', + content='We should work on this, really', + user='pingou', + ticketfolder=os.path.join(self.path, 'tickets') + ) + self.session.commit() + self.assertEqual(msg.title, 'Test issue #2') + + # Add a comment to an issue + issue = pagure.lib.search_issues(self.session, repo, issueid=1) + msg = pagure.lib.add_issue_comment( + session=self.session, + issue=issue, + comment='Hey look a comment!', + user='foo', + ticketfolder=None + ) + self.session.commit() + self.assertEqual(msg, 'Comment added') + + # add pull-requests + req = pagure.lib.new_pull_request( + session=self.session, + repo_from=repo, + branch_from='feature', + repo_to=repo, + branch_to='master', + title='test pull-request', + user='pingou', + requestfolder=os.path.join(self.path, 'requests'), + ) + self.session.commit() + self.assertEqual(req.id, 3) + self.assertEqual(req.title, 'test pull-request') + + req = pagure.lib.new_pull_request( + session=self.session, + repo_from=repo, + branch_from='feature2', + repo_to=repo, + branch_to='master', + title='test pull-request', + user='pingou', + requestfolder=os.path.join(self.path, 'requests'), + ) + self.session.commit() + self.assertEqual(req.id, 4) + self.assertEqual(req.title, 'test pull-request') + + # Add comment on a pull-request + request = pagure.lib.search_pull_requests( + self.session, requestid=3) + + msg = pagure.lib.add_pull_request_comment( + session=self.session, + request=request, + commit='commithash', + tree_id=None, + filename='file', + row=None, + comment='This is awesome, I got to remember it!', + user='foo', + requestfolder=None, + ) + self.assertEqual(msg, 'Comment added') # Check before deleting the project output = self.app.get('/') @@ -3035,17 +3032,16 @@ index 0000000..fb7093d 'Forks 0', output.data) - with patch('pagure.ensure_lock'): - # add user - repo = pagure.get_authorized_project(self.session, 'test') - msg = pagure.lib.add_user_to_project( - session=self.session, - project=repo, - new_user='foo', - user='pingou', - ) - self.session.commit() - self.assertEqual(msg, 'User added') + # add user + repo = pagure.get_authorized_project(self.session, 'test') + msg = pagure.lib.add_user_to_project( + session=self.session, + project=repo, + new_user='foo', + user='pingou', + ) + self.session.commit() + self.assertEqual(msg, 'User added') # Check before deleting the project output = self.app.get('/') @@ -3116,35 +3112,34 @@ index 0000000..fb7093d 'Forks 0', output.data) - with patch('pagure.ensure_lock'): - # Create group - msg = pagure.lib.add_group( - self.session, - group_name='foo', - display_name='foo group', - description=None, - group_type='bar', - user='pingou', - is_admin=False, - blacklist=[], - ) - self.session.commit() - self.assertEqual(msg, 'User `pingou` added to the group `foo`.') - - # Add group to the project - repo = pagure.get_authorized_project(self.session, 'test') - msg = pagure.lib.add_group_to_project( - session=self.session, - project=repo, - new_group='foo', - user='pingou', - ) - self.session.commit() - self.assertEqual(msg, 'Group added') - - # check if group where we expect it - repo = pagure.get_authorized_project(self.session, 'test') - self.assertEqual(len(repo.projects_groups), 1) + # Create group + msg = pagure.lib.add_group( + self.session, + group_name='foo', + display_name='foo group', + description=None, + group_type='bar', + user='pingou', + is_admin=False, + blacklist=[], + ) + self.session.commit() + self.assertEqual(msg, 'User `pingou` added to the group `foo`.') + + # Add group to the project + repo = pagure.get_authorized_project(self.session, 'test') + msg = pagure.lib.add_group_to_project( + session=self.session, + project=repo, + new_group='foo', + user='pingou', + ) + self.session.commit() + self.assertEqual(msg, 'Group added') + + # check if group where we expect it + repo = pagure.get_authorized_project(self.session, 'test') + self.assertEqual(len(repo.projects_groups), 1) # Check before deleting the project output = self.app.get('/') @@ -3211,31 +3206,30 @@ index 0000000..fb7093d 'Forks 0', output.data) - with patch('pagure.ensure_lock'): - # Create the issue - repo = pagure.get_authorized_project(self.session, 'test') - msg = pagure.lib.new_issue( - session=self.session, - repo=repo, - title='Test issue', - content='We should work on this', - user='pingou', - ticketfolder=os.path.join(self.path, 'tickets') - ) - self.session.commit() - self.assertEqual(msg.title, 'Test issue') - - # Add a tag to the issue - repo = pagure.get_authorized_project(self.session, 'test') - issue = pagure.lib.search_issues(self.session, repo, issueid=1) - msg = pagure.lib.add_tag_obj( - session=self.session, - obj=issue, - tags='tag1', - user='pingou', - ticketfolder=None) - self.session.commit() - self.assertEqual(msg, 'Issue tagged with: tag1') + # Create the issue + repo = pagure.get_authorized_project(self.session, 'test') + msg = pagure.lib.new_issue( + session=self.session, + repo=repo, + title='Test issue', + content='We should work on this', + user='pingou', + ticketfolder=os.path.join(self.path, 'tickets') + ) + self.session.commit() + self.assertEqual(msg.title, 'Test issue') + + # Add a tag to the issue + repo = pagure.get_authorized_project(self.session, 'test') + issue = pagure.lib.search_issues(self.session, repo, issueid=1) + msg = pagure.lib.add_tag_obj( + session=self.session, + obj=issue, + tags='tag1', + user='pingou', + ticketfolder=None) + self.session.commit() + self.assertEqual(msg, 'Issue tagged with: tag1') # Check before deleting the project output = self.app.get('/') diff --git a/tests/test_pagure_lib_git.py b/tests/test_pagure_lib_git.py index f87659c..604aff3 100644 --- a/tests/test_pagure_lib_git.py +++ b/tests/test_pagure_lib_git.py @@ -1298,9 +1298,8 @@ index 9f44358..2a552bb 100644 patch = '\n'.join(npatch) self.assertEqual(patch, exp) - @patch('pagure.ensure_lock') @patch('pagure.lib.notify.send_email') - def test_update_git(self, elock, email_f): + def test_update_git(self, email_f): """ Test the update_git of pagure.lib.git. """ email_f.return_value = True @@ -1518,8 +1517,7 @@ index 458821a..77674a8 #print patch self.assertEqual(patch, exp) - @patch('pagure.ensure_lock') - def test_clean_git(self, elock): + def test_clean_git(self): """ Test the clean_git method of pagure.lib.git. """ pagure.lib.git.clean_git(None, None, None) @@ -1550,9 +1548,8 @@ index 458821a..77674a8 files = [entry.name for entry in commit.tree] self.assertEqual(files, []) - @patch('pagure.ensure_lock') @patch('pagure.lib.notify.send_email') - def test_update_git_requests(self, elock, email_f): + def test_update_git_requests(self, email_f): """ Test the update_git of pagure.lib.git for pull-requests. """ email_f.return_value = True @@ -2393,8 +2390,7 @@ index 0000000..60f7480 'test request to namespaced repo' ) - @patch('pagure.ensure_lock') - def test_read_git_lines(self, elock): + def test_read_git_lines(self): """ Test the read_git_lines method of pagure.lib.git. """ self.test_update_git() @@ -2417,8 +2413,7 @@ index 0000000..60f7480 output[0].endswith(": Test issue'\n") ) - @patch('pagure.ensure_lock') - def test_get_revs_between(self, elock): + def test_get_revs_between(self): """ Test the get_revs_between method of pagure.lib.git. """ self.test_update_git() @@ -2485,8 +2480,7 @@ index 0000000..60f7480 '0', branch_commit.oid.hex, gitrepo, 'refs/heads/feature') self.assertEqual(output4, [branch_commit.oid.hex]) - @patch('pagure.ensure_lock') - def test_get_author(self, elock): + def test_get_author(self): """ Test the get_author method of pagure.lib.git. """ self.test_update_git() From 8e1ee656991ddb5576aad16b415ac9921ad4de72 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 13/57] Update_git is now async Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 94973a1..77bde16 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -206,6 +206,26 @@ def _generate_gitolite_acls(): def update_git(obj, repo, repofolder): + """ Schedules an update_repo task after determining arguments. """ + user = None + ticketuid = None + requestuid = None + if obj.isa == 'issue': + ticketuid = obj.uid + elif obj.isa == 'pull-request': + args['requestuid'] = obj.uid + requestuid = obj.uid + else: + raise NotImplementedError('Unknown object type %s' % obj.isa) + + if repo.is_fork: + user = repo.user + + return pagure.lib.tasks.update_git.delay( + repo.name, repo.namespace, user, ticketuid, requestuid) + + +def _update_git(obj, repo, repofolder): """ Update the given issue in its git. This method forks the provided repo, add/edit the issue whose file name @@ -218,8 +238,6 @@ def update_git(obj, repo, repofolder): if not repofolder: return - pagure.ensure_lock(repo) - # Get the fork repopath = os.path.join(repofolder, repo.path) @@ -306,8 +324,6 @@ def clean_git(obj, repo, repofolder): if not repofolder: return - pagure.ensure_lock(repo) - _log.info('Update the git repo: %s to remove: %s', repo.path, obj) # Get the fork @@ -819,8 +835,6 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): if not ticketfolder: return - pagure.ensure_lock(repo) - # Prefix the filename with a timestamp: filename = '%s-%s' % ( hashlib.sha256(filestream.read()).hexdigest(), @@ -927,8 +941,6 @@ def update_file_in_git( ''' _log.info('Updating file: %s in the repo: %s', filename, repo.path) - pagure.ensure_lock(repo) - # Get the fork repopath = pagure.get_repo_path(repo) diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 151e744..5fba456 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -133,3 +133,19 @@ def create_project(username, namespace, name, add_readme, ignore_existing_repo): generate_gitolite_acls.delay() return ret('view_repo', repo=name, namespace=namespace) + + +@conn.task +def update_git(name, namespace, user, ticketuid=None, requestuid=None): + project = pagure.lib._get_project(pagure.SESSION, namespace=namespace, + name=name, user=user, with_lock=True) + if ticketuid is not None: + obj = pagure.lib.get_issue_by_uid(pagure.SESSION, ticketuid) + folder = os.path.join(APP.config['TICKETS_FOLDER'], project.path) + elif requestuid is not None: + obj = pagure.lib.get_request_by_uid(pagure.SESSION, requestuid) + folder = os.path.join(APP.config['REQUESTS_FOLDER'], project.path) + else: + raise NotImplementedError('No ticket ID or request ID provided') + + return pagure.lib.git._update_git(obj, project, folder) From 5c446b23b6d5206eb0b991a6e7f5b7e292413903 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 14/57] Do not allow easy access to locking Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 98f2705..f8a49a7 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -354,8 +354,7 @@ def is_repo_user(repo_obj): ) or (user in usergrps) -def get_authorized_project(session, project_name, user=None, namespace=None, - with_lock=False): +def get_authorized_project(session, project_name, user=None, namespace=None): ''' Retrieving the project with user permission constraint :arg session: The SQLAlchemy session to use @@ -371,17 +370,7 @@ def get_authorized_project(session, project_name, user=None, namespace=None, :rtype: Project ''' - if with_lock: - if not authenticated(): - logger.info('Unauthenticated request requested lock') - with_lock = False - - if not flask.request.method == 'POST': - logger.info('non-POST request requested lock') - with_lock = False - - repo = pagure.lib._get_project(session, project_name, user, namespace, - with_lock) + repo = pagure.lib._get_project(session, project_name, user, namespace) if repo and repo.private and not is_repo_admin(repo): return None @@ -461,7 +450,7 @@ def set_session(): @APP.before_request -def set_variables(with_lock=False): +def set_variables(): """ This method retrieves the repo and username set in the URLs and provides some of the variables that are most often used. """ @@ -482,9 +471,7 @@ def set_variables(with_lock=False): # endpoint called is part of the API, just don't do anything if repo: flask.g.repo = pagure.get_authorized_project( - SESSION, repo, user=username, namespace=namespace, - with_lock=with_lock) - flask.g.repo_locked = with_lock + SESSION, repo, user=username, namespace=namespace) if authenticated(): flask.g.repo_forked = pagure.get_authorized_project( SESSION, repo, user=flask.g.fas_user.username, From fe56181e495a1038e673b3bc05f553de88fd4c15 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 15/57] Reuse sessionmaker's between create_session calls Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index c997b60..86d0847 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -89,6 +89,7 @@ def get_user(session, key): return user_obj +SESSIONMAKER = None def create_session(db_url, debug=False, pool_recycle=3600): ''' Create the Session object to use to query the database. @@ -101,15 +102,20 @@ def create_session(db_url, debug=False, pool_recycle=3600): :return a Session that can be used to query the database. ''' - if db_url.startswith('postgres'): # pragma: no cover - engine = sqlalchemy.create_engine( - db_url, echo=debug, pool_recycle=pool_recycle, - client_encoding='utf8') - else: # pragma: no cover - engine = sqlalchemy.create_engine( - db_url, echo=debug, pool_recycle=pool_recycle) - scopedsession = scoped_session(sessionmaker(bind=engine)) - model.BASE.metadata.bind = scopedsession + global SESSIONMAKER + + if SESSIONMAKER is None: + if db_url.startswith('postgres'): # pragma: no cover + engine = sqlalchemy.create_engine( + db_url, echo=debug, pool_recycle=pool_recycle, + client_encoding='utf8') + else: # pragma: no cover + engine = sqlalchemy.create_engine( + db_url, echo=debug, pool_recycle=pool_recycle) + SESSIONMAKER = sessionmaker(bind=engine) + + scopedsession = scoped_session(SESSIONMAKER) + # model.BASE.metadata.bind = scopedsession return scopedsession From 241f0e11fab54b2c251b9198ba50bec86c4edf73 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 16/57] API should not lock Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/__init__.py b/pagure/api/__init__.py index f94ef66..f600bc5 100644 --- a/pagure/api/__init__.py +++ b/pagure/api/__init__.py @@ -87,12 +87,10 @@ class APIERROR(enum.Enum): ENOGROUP = 'Group not found' -def get_authorized_api_project(SESSION, repo, user=None, namespace=None, - with_lock=False): +def get_authorized_api_project(SESSION, repo, user=None, namespace=None): ''' Helper function to get an authorized_project with optional lock. ''' repo = pagure.get_authorized_project( - SESSION, repo, user=user, namespace=namespace, with_lock=with_lock) - flask.g.repo_locked = with_lock + SESSION, repo, user=user, namespace=namespace) flask.g.repo = repo return repo diff --git a/pagure/api/ci/jenkins.py b/pagure/api/ci/jenkins.py index 88f4210..e539ab6 100644 --- a/pagure/api/ci/jenkins.py +++ b/pagure/api/ci/jenkins.py @@ -45,7 +45,7 @@ def jenkins_ci_notification( """ project = pagure.lib._get_project( - SESSION, repo, user=username, namespace=namespace, with_lock=True) + SESSION, repo, user=username, namespace=namespace) flask.g.repo_locked = True flask.g.repo = project if not project: diff --git a/pagure/api/issue.py b/pagure/api/issue.py index 31dcfe6..140dc88 100644 --- a/pagure/api/issue.py +++ b/pagure/api/issue.py @@ -27,7 +27,7 @@ from pagure.api import ( ) -def _get_repo(repo_name, username=None, namespace=None, with_lock=False): +def _get_repo(repo_name, username=None, namespace=None): """Check if repository exists and get repository name :param repo_name: name of repository :param username: @@ -37,8 +37,7 @@ def _get_repo(repo_name, username=None, namespace=None, with_lock=False): :return: repository name """ repo = get_authorized_api_project( - SESSION, repo_name, user=username, namespace=namespace, - with_lock=with_lock) + SESSION, repo_name, user=username, namespace=namespace) if repo is None: raise pagure.exceptions.APIError( @@ -194,7 +193,7 @@ def api_new_issue(repo, username=None, namespace=None): """ output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) if flask.g.token.project and repo != flask.g.token.project: @@ -678,7 +677,7 @@ def api_change_status_issue(repo, issueid, username=None, namespace=None): """ output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo, project_token=False) @@ -796,7 +795,7 @@ def api_change_milestone_issue(repo, issueid, username=None, namespace=None): """ # noqa output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo) @@ -895,7 +894,7 @@ def api_comment_issue(repo, issueid, username=None, namespace=None): """ output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo, project_token=False) @@ -975,7 +974,7 @@ def api_assign_issue(repo, issueid, username=None, namespace=None): """ output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo) @@ -1073,7 +1072,7 @@ def api_subscribe_issue(repo, issueid, username=None, namespace=None): """ # noqa output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo) @@ -1156,7 +1155,7 @@ def api_update_custom_field( """ # noqa output = {} - repo = _get_repo(repo, username, namespace, with_lock=True) + repo = _get_repo(repo, username, namespace) _check_issue_tracker(repo) _check_token(repo) From 120396afa6656f3af0692b37e6b63803c73a9a45 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 17/57] Use new session for each action Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 5fba456..92f1dfa 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -47,9 +47,11 @@ def generate_gitolite_acls(): @conn.task def create_project(username, namespace, name, add_readme, ignore_existing_repo): - project = pagure.lib._get_project(pagure.SESSION, namespace=namespace, + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, name=name, with_lock=True) - userobj = pagure.lib.search_user(pagure.SESSION, username=username) + userobj = pagure.lib.search_user(session, username=username) gitrepo = os.path.join(APP.config['GIT_FOLDER'], project.path) # Add the readme file if it was asked @@ -125,27 +127,32 @@ def create_project(username, namespace, name, add_readme, ignore_existing_repo): #dbobj = plugin.db_object() #dbobj.active = True #dbobj.project_id = project.id - #pagure.SESSION.add(dbobj) - #pagure.SESSION.flush() + #session.add(dbobj) + #session.flush() #plugin.set_up(project) #plugin.install(project, dbobj) - #pagure.SESSION.commit() + #session.commit() + session.remove() generate_gitolite_acls.delay() return ret('view_repo', repo=name, namespace=namespace) @conn.task def update_git(name, namespace, user, ticketuid=None, requestuid=None): - project = pagure.lib._get_project(pagure.SESSION, namespace=namespace, + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, name=name, user=user, with_lock=True) if ticketuid is not None: - obj = pagure.lib.get_issue_by_uid(pagure.SESSION, ticketuid) - folder = os.path.join(APP.config['TICKETS_FOLDER'], project.path) + obj = pagure.lib.get_issue_by_uid(session, ticketuid) + folder = APP.config['TICKETS_FOLDER'] elif requestuid is not None: - obj = pagure.lib.get_request_by_uid(pagure.SESSION, requestuid) - folder = os.path.join(APP.config['REQUESTS_FOLDER'], project.path) + obj = pagure.lib.get_request_by_uid(session, requestuid) + folder = APP.config['REQUESTS_FOLDER'] else: raise NotImplementedError('No ticket ID or request ID provided') - return pagure.lib.git._update_git(obj, project, folder) + result = pagure.lib.git._update_git(obj, project, folder) + session.remove() + return result From 601b7f499574538db7ad04005c302819e37ca3a7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 18/57] Make db_url optional Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 86d0847..9e4be8f 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -90,7 +90,7 @@ def get_user(session, key): SESSIONMAKER = None -def create_session(db_url, debug=False, pool_recycle=3600): +def create_session(db_url=None, debug=False, pool_recycle=3600): ''' Create the Session object to use to query the database. :arg db_url: URL used to connect to the database. The URL contains @@ -105,6 +105,8 @@ def create_session(db_url, debug=False, pool_recycle=3600): global SESSIONMAKER if SESSIONMAKER is None: + if db_url is None: + raise ValueError("First call to create_session needs db_url") if db_url.startswith('postgres'): # pragma: no cover engine = sqlalchemy.create_engine( db_url, echo=debug, pool_recycle=pool_recycle, From 189fa8c3f1b766ef27536fbda31596824a534310 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 19/57] Taskify clean_git Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 77bde16..3a7f96e 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -213,7 +213,6 @@ def update_git(obj, repo, repofolder): if obj.isa == 'issue': ticketuid = obj.uid elif obj.isa == 'pull-request': - args['requestuid'] = obj.uid requestuid = obj.uid else: raise NotImplementedError('Unknown object type %s' % obj.isa) @@ -317,6 +316,17 @@ def _update_git(obj, repo, repofolder): def clean_git(obj, repo, repofolder): + user = None + ticketuid = obj.uid + + if repo.is_fork: + user = repo.user + + return pagure.lib.tasks.clean_git.delay( + repo.name, repo.namespace, user, ticketuid) + + +def _clean_git(obj, repo, repofolder): """ Update the given issue remove it from its git. """ diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 92f1dfa..3459e83 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -142,8 +142,8 @@ def create_project(username, namespace, name, add_readme, ignore_existing_repo): def update_git(name, namespace, user, ticketuid=None, requestuid=None): session = pagure.lib.create_session() - project = pagure.lib._get_project(session, namespace=namespace, - name=name, user=user, with_lock=True) + project = pagure.lib._get_project(session, namespace=namespace, name=name, + user=user, with_lock=True) if ticketuid is not None: obj = pagure.lib.get_issue_by_uid(session, ticketuid) folder = APP.config['TICKETS_FOLDER'] @@ -153,6 +153,26 @@ def update_git(name, namespace, user, ticketuid=None, requestuid=None): else: raise NotImplementedError('No ticket ID or request ID provided') + if obj is None: + raise Exception('Unable to find object') + result = pagure.lib.git._update_git(obj, project, folder) session.remove() return result + + +@conn.task +def clean_git(name, namespace, user, ticketuid): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, name=name, + user=user, with_lock=True) + obj = pagure.lib.get_issue_by_uid(session, ticketuid) + folder = APP.config['TICKETS_FOLDER'] + + if obj is None: + raise Exception('Unable to find object') + + result = pagure.lib.git._clean_git(obj, project, folder) + session.remove() + return result From 8cb22493c50d42ac29c8911c3baa0d0b73ade46f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 20/57] Taskify delete branch and edit file Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 3a7f96e..98b91b5 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -937,7 +937,7 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): return os.path.join('files', filename) -def update_file_in_git( +def _update_file_in_git( repo, branch, branchto, filename, content, message, user, email): ''' Update a specific file in the specified repository with the content given and commit the change under the user's name. diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 3459e83..eb39af4 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -176,3 +176,38 @@ def clean_git(name, namespace, user, ticketuid): result = pagure.lib.git._clean_git(obj, project, folder) session.remove() return result + + +@conn.task +def update_file_in_git(name, namespace, user, branch, branchto, filename, + content, message, username, email): + session = pagure.lib.create_session() + + userobj = pagure.lib.search_user(session, username=username) + project = pagure.lib._get_project(session, namespace=namespace, name=name, + user=user, with_lock=True) + + pagure.lib.git._update_file_in_git(project, branch, branchto, filename, + content, message, userobj, email) + + session.remove() + return ret('view_commits', repo=project.name, username=user, + namespace=namespace, branchname=branchto) + + +@conn.task +def delete_branch(name, namespace, user, branchname): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, name=name, + user=user, with_lock=True) + repo_obj = pygit2.Repository(pagure.get_repo_path(project)) + + try: + branch = repo_obj.lookup_branch(branchname) + branch.delete() + except pygit2.GitError as err: + _log.exception(err) + + session.remove() + return ret('view_repo', repo=name, namespace=namespace, username=user) diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 5e4acd5..c87e326 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -2123,8 +2123,10 @@ def edit_file(repo, branchname, filename, username=None, namespace=None): if form.validate_on_submit(): try: - pagure.lib.git.update_file_in_git( - repo, + taskid = pagure.lib.tasks.update_file_in_git.delay( + repo.name, + repo.namespace, + repo.user if repo.is_fork else None, branch=branchname, branchto=form.branch.data, filename=filename, @@ -2133,15 +2135,11 @@ def edit_file(repo, branchname, filename, username=None, namespace=None): form.commit_title.data.strip(), form.commit_message.data.strip() ), - user=flask.g.fas_user, + username=flask.g.fas_user.username, email=form.email.data, - ) - flask.flash('Changes committed') - return flask.redirect( - flask.url_for( - '.view_commits', repo=repo.name, username=username, - namespace=namespace, branchname=form.branch.data) - ) + ).id + return flask.redirect(flask.url_for( + 'wait_task', taskid=taskid)) except pagure.exceptions.PagureException as err: # pragma: no cover _log.exception(err) flask.flash('Commit could not be done', 'error') @@ -2195,16 +2193,10 @@ def delete_branch(repo, branchname, username=None, namespace=None): if branchname not in repo_obj.listall_branches(): flask.abort(404, 'Branch not found') - try: - branch = repo_obj.lookup_branch(branchname) - branch.delete() - flask.flash('Branch `%s` deleted' % branchname) - except pygit2.GitError as err: - _log.exception(err) - flask.flash('Could not delete `%s`' % branchname, 'error') - + taskid = pagure.lib.tasks.delete_branch.delay(repo, namespace, username, + branchname).id return flask.redirect(flask.url_for( - 'view_repo', repo=repo, username=username, namespace=namespace)) + 'wait_task', taskid=taskid)) @APP.route('/docs//') From ac056b540984b5ffeee06f253b835b55134dc846 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 21/57] Taskified forking Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 9e4be8f..345520d 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1776,66 +1776,12 @@ def fork_project(session, user, repo, gitfolder, session.add(project) # Make sure we won't have SQLAlchemy error before we create the repo session.flush() + session.commit() - frepo = pygit2.clone_repository(reponame, forkreponame, bare=True) - # Clone all the branches as well - for branch in frepo.listall_branches(pygit2.GIT_BRANCH_REMOTE): - branch_obj = frepo.lookup_branch(branch, pygit2.GIT_BRANCH_REMOTE) - name = branch_obj.branch_name.replace( - branch_obj.remote_name, '', 1)[1:] - if name in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): - continue - frepo.create_branch(name, frepo.get(branch_obj.target.hex)) - - # Create the git-daemin-export-ok file on the clone - http_clone_file = os.path.join(forkreponame, 'git-daemon-export-ok') - if not os.path.exists(http_clone_file): - with open(http_clone_file, 'w'): - pass - - docrepo = os.path.join(docfolder, project.path) - if os.path.exists(docrepo): - shutil.rmtree(forkreponame) - raise pagure.exceptions.RepoExistsException( - 'The docs "%s" already exists' % project.path - ) - pygit2.init_repository(docrepo, bare=True) - - ticketrepo = os.path.join(ticketfolder, project.path) - if os.path.exists(ticketrepo): - shutil.rmtree(forkreponame) - shutil.rmtree(docrepo) - raise pagure.exceptions.RepoExistsException( - 'The tickets repo "%s" already exists' % project.path - ) - pygit2.init_repository( - ticketrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - requestrepo = os.path.join(requestfolder, project.path) - if os.path.exists(requestrepo): - shutil.rmtree(forkreponame) - shutil.rmtree(docrepo) - shutil.rmtree(ticketrepo) - raise pagure.exceptions.RepoExistsException( - 'The requests repo "%s" already exists' % project.path - ) - pygit2.init_repository( - requestrepo, bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) - - pagure.lib.notify.log( - project, - topic='project.forked', - msg=dict( - project=project.to_json(public=True), - agent=user_obj.username, - ), - ) - - return 'Repo "{0}" cloned to "{1}/{0}"'.format( - '%s/%s' % (repo.namespace, repo.name) if repo.namespace else repo.name, - user) + return tasks.fork.delay(repo.name, + repo.namespace, + repo.user if repo.is_fork else None, + user).id def search_projects( diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index eb39af4..243308b 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -211,3 +211,77 @@ def delete_branch(name, namespace, user, branchname): session.remove() return ret('view_repo', repo=name, namespace=namespace, username=user) + + +@conn.task +def fork(name, namespace, user_owner, user_forker): + session = pagure.lib.create_session() + + repo_from = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user_owner) + repo_to = pagure.lib._get_project(session, namespace=namespace, name=name, + user=user_forker, with_lock=True) + + reponame = os.path.join(APP.config['GIT_FOLDER'], repo_from.path) + forkreponame = os.path.join(APP.config['GIT_FOLDER'], repo_to.path) + + frepo = pygit2.clone_repository(reponame, forkreponame, bare=True) + # Clone all the branches as well + for branch in frepo.listall_branches(pygit2.GIT_BRANCH_REMOTE): + branch_obj = frepo.lookup_branch(branch, pygit2.GIT_BRANCH_REMOTE) + name = branch_obj.branch_name.replace( + branch_obj.remote_name, '', 1)[1:] + if name in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): + continue + frepo.create_branch(name, frepo.get(branch_obj.target.hex)) + + # Create the git-daemon-export-ok file on the clone + http_clone_file = os.path.join(forkreponame, 'git-daemon-export-ok') + if not os.path.exists(http_clone_file): + with open(http_clone_file, 'w'): + pass + + docrepo = os.path.join(APP.config['DOCS_FOLDER'], repo_to.path) + if os.path.exists(docrepo): + shutil.rmtree(forkreponame) + raise pagure.exceptions.RepoExistsException( + 'The docs "%s" already exists' % repo_to.path + ) + pygit2.init_repository(docrepo, bare=True) + + ticketrepo = os.path.join(APP.config['TICKETS_FOLDER'], repo_to.path) + if os.path.exists(ticketrepo): + shutil.rmtree(forkreponame) + shutil.rmtree(docrepo) + raise pagure.exceptions.RepoExistsException( + 'The tickets repo "%s" already exists' % repo_to.path + ) + pygit2.init_repository( + ticketrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + requestrepo = os.path.join(APP.config['REQUESTS_FOLDER'], repo_to.path) + if os.path.exists(requestrepo): + shutil.rmtree(forkreponame) + shutil.rmtree(docrepo) + shutil.rmtree(ticketrepo) + raise pagure.exceptions.RepoExistsException( + 'The requests repo "%s" already exists' % repo_to.path + ) + pygit2.init_repository( + requestrepo, bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) + + pagure.lib.notify.log( + repo_to, + topic='project.forked', + msg=dict( + project=repo_to.to_json(public=True), + agent=user_forker, + ), + ) + + session.remove() + generate_gitolite_acls.delay() + return ret('view_repo', repo=name, namespace=namespace, + username=user_forker) diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 31dc14a..4338cbd 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -881,7 +881,7 @@ def fork_project(repo, username=None, namespace=None): namespace=namespace)) try: - message = pagure.lib.fork_project( + taskid = pagure.lib.fork_project( session=SESSION, repo=repo, gitfolder=APP.config['GIT_FOLDER'], @@ -891,15 +891,8 @@ def fork_project(repo, username=None, namespace=None): user=flask.g.fas_user.username) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() - flask.flash(message) - return flask.redirect( - flask.url_for( - 'view_repo', - username=flask.g.fas_user.username, - namespace=namespace, - repo=repo.name) - ) + return flask.redirect(flask.url_for( + 'wait_task', taskid=taskid)) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover From 2a3bd8ef4b24a46925046bdf2a8409fc479e9120 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 22/57] Taskify fork and edit Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 345520d..92b438d 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1743,7 +1743,8 @@ def update_user_settings(session, settings, user): def fork_project(session, user, repo, gitfolder, - docfolder, ticketfolder, requestfolder): + docfolder, ticketfolder, requestfolder, + editbranch=None, editfile=None): ''' Fork a given project into the user's forks. ''' reponame = os.path.join(gitfolder, repo.path) forkreponame = '%s.git' % os.path.join( @@ -1781,7 +1782,9 @@ def fork_project(session, user, repo, gitfolder, return tasks.fork.delay(repo.name, repo.namespace, repo.user if repo.is_fork else None, - user).id + user, + editbranch, + editfile).id def search_projects( diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 243308b..63b97ef 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -214,7 +214,7 @@ def delete_branch(name, namespace, user, branchname): @conn.task -def fork(name, namespace, user_owner, user_forker): +def fork(name, namespace, user_owner, user_forker, editbranch, editfile): session = pagure.lib.create_session() repo_from = pagure.lib._get_project(session, namespace=namespace, @@ -229,11 +229,11 @@ def fork(name, namespace, user_owner, user_forker): # Clone all the branches as well for branch in frepo.listall_branches(pygit2.GIT_BRANCH_REMOTE): branch_obj = frepo.lookup_branch(branch, pygit2.GIT_BRANCH_REMOTE) - name = branch_obj.branch_name.replace( + branchname = branch_obj.branch_name.replace( branch_obj.remote_name, '', 1)[1:] - if name in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): + if branchname in frepo.listall_branches(pygit2.GIT_BRANCH_LOCAL): continue - frepo.create_branch(name, frepo.get(branch_obj.target.hex)) + frepo.create_branch(branchname, frepo.get(branch_obj.target.hex)) # Create the git-daemon-export-ok file on the clone http_clone_file = os.path.join(forkreponame, 'git-daemon-export-ok') @@ -283,5 +283,11 @@ def fork(name, namespace, user_owner, user_forker): session.remove() generate_gitolite_acls.delay() - return ret('view_repo', repo=name, namespace=namespace, - username=user_forker) + + if editfile is None: + return ret('view_repo', repo=name, namespace=namespace, + username=user_forker) + else: + return ret('edit_file', repo=name, namespace=namespace, + username=user_forker, branchname=editbranch, + filename=editfile) diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 4338cbd..811ff0c 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -1238,26 +1238,20 @@ def fork_edit_file( )) try: - message = pagure.lib.fork_project( + taskid = pagure.lib.fork_project( session=SESSION, repo=repo, gitfolder=APP.config['GIT_FOLDER'], docfolder=APP.config['DOCS_FOLDER'], ticketfolder=APP.config['TICKETS_FOLDER'], requestfolder=APP.config['REQUESTS_FOLDER'], - user=flask.g.fas_user.username) + user=flask.g.fas_user.username, + editbranch=branchname, + editfile=filename) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() - flask.flash(message) return flask.redirect(flask.url_for( - 'edit_file', - username=flask.g.fas_user.username, - namespace=repo.namespace, - repo=repo.name, - branchname=branchname, - filename=filename - )) + 'wait_task', taskid=taskid)) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover From fb0175fe1610475ba4f1a2b34dd9857613c16900 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 23/57] Fix update_file_in_git task Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index c87e326..c464a1c 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -2126,7 +2126,7 @@ def edit_file(repo, branchname, filename, username=None, namespace=None): taskid = pagure.lib.tasks.update_file_in_git.delay( repo.name, repo.namespace, - repo.user if repo.is_fork else None, + repo.user.username if repo.is_fork else None, branch=branchname, branchto=form.branch.data, filename=filename, From eb93951617fd73c1c9e35dd1126e3cddb272af33 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 24/57] Make sure to pass username everywhere Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 92b438d..8f4d0d9 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1781,7 +1781,7 @@ def fork_project(session, user, repo, gitfolder, return tasks.fork.delay(repo.name, repo.namespace, - repo.user if repo.is_fork else None, + repo.user.username if repo.is_fork else None, user, editbranch, editfile).id diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 98b91b5..18d28eb 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -217,11 +217,10 @@ def update_git(obj, repo, repofolder): else: raise NotImplementedError('Unknown object type %s' % obj.isa) - if repo.is_fork: - user = repo.user - return pagure.lib.tasks.update_git.delay( - repo.name, repo.namespace, user, ticketuid, requestuid) + repo.name, repo.namespace, + repo.user.username if repo.is_fork else None, + ticketuid, requestuid) def _update_git(obj, repo, repofolder): @@ -323,7 +322,7 @@ def clean_git(obj, repo, repofolder): user = repo.user return pagure.lib.tasks.clean_git.delay( - repo.name, repo.namespace, user, ticketuid) + repo.name, repo.namespace, user.username, ticketuid) def _clean_git(obj, repo, repofolder): From fb4f421c1a86b58bddf686af0d5e9c39b8030f2c Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 25/57] Ignore this Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 63b97ef..77bb254 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -122,7 +122,7 @@ def create_project(username, namespace, name, add_readme, ignore_existing_repo): mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) # Install the default hook - # TODO: This needs enabling, but doesn't work in my env + # TODO: This needs enabling, but doesn't work in my env due to tables #plugin = pagure.lib.plugins.get_plugin('default') #dbobj = plugin.db_object() #dbobj.active = True From c0c627230008aeea55c81cac96f2ee4c740cfa9d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 26/57] Taskify remote PR Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/__init__.py b/pagure/__init__.py index f8a49a7..4f134c3 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -658,7 +658,7 @@ def get_repo_path(repo): return repopath -def get_remote_repo_path(remote_git, branch_from, loop=False): +def get_remote_repo_path(remote_git, branch_from, ignore_non_exist=False): """ Return the path of the remote git repository corresponding to the provided information. """ @@ -667,36 +667,26 @@ def get_remote_repo_path(remote_git, branch_from, loop=False): werkzeug.secure_filename('%s_%s' % (remote_git, branch_from)) ) - if not os.path.exists(repopath): - try: - pygit2.clone_repository( - remote_git, repopath, checkout_branch=branch_from) - except Exception as err: - APP.logger.exception(err) - flask.abort( - 500, - 'The following error was raised when trying to clone the ' - 'remote repo: %s' % str(err) - ) + if not os.path.exists(repopath) and not ignore_non_exist: + return None else: - repo = pagure.lib.repo.PagureRepo(repopath) - try: - repo.pull(branch=branch_from, force=True) - except pygit2.GitError as err: - APP.logger.debug( - 'Error pull the repo: %s -- error: %s' % (repopath, err)) - if str(err).lower() != 'no content-type header in response': - APP.logger.exception(err) - flask.abort( - 500, - 'The following error was raised when trying to pull the ' - 'changes from the remote: %s' % str(err) - ) - except pagure.exceptions.PagureException as err: - APP.logger.exception(err) - flask.abort(500, str(err)) + return repopath - return repopath + +def wait_for_task(taskid): + return flask.redirect(flask.url_for( + 'wait_task', taskid=taskid)) + + +def wait_for_task_post(taskid, form, endpoint, initial=False, **kwargs): + form_action = flask.url_for(endpoint, **kwargs) + return flask.render_template( + 'waiting_post.html', + taskid=taskid, + form_action=form_action, + form_data=form.data, + csrf=form.csrf_token, + initial=initial) ip_middle_octet = u"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5]))" diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 77bb254..491e338 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -291,3 +291,19 @@ def fork(name, namespace, user_owner, user_forker, editbranch, editfile): return ret('edit_file', repo=name, namespace=namespace, username=user_forker, branchname=editbranch, filename=editfile) + + +@conn.task +def pull_remote_repo(name, namespace, user, remote_git, branch_from, + branch_to): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user) + + clonepath = pagure.get_remote_repo_path(remote_git, branch_from, + ignore_non_exist=True) + pygit2.clone_repository( + remote_git, clonepath, checkout_branch=branch_from) + + session.remove() diff --git a/pagure/templates/waiting.html b/pagure/templates/waiting.html index 6e4ff18..2a71ea2 100644 --- a/pagure/templates/waiting.html +++ b/pagure/templates/waiting.html @@ -4,7 +4,7 @@ {% block header %} -{% endblock %} +{% endblock %} {% block content %} diff --git a/pagure/templates/waiting_post.html b/pagure/templates/waiting_post.html new file mode 100644 index 0000000..95612c6 --- /dev/null +++ b/pagure/templates/waiting_post.html @@ -0,0 +1,40 @@ +{% extends "master.html" %} + +{% block title %}Waiting{% endblock %} + +{% block content %} + +

+ Waiting +

+ +

+ We are waiting for your task to finish. +

+ + {% for field in form_data %} + + {% endfor %} + {{ csrf }} + This page should be refreshed automatically, but if not click . +
+

+ +{% endblock %} + +{% block jscripts %} + {{ super() }} + +{% endblock %} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 19f7d66..84c98cc 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -474,8 +474,7 @@ def new_project(): user_ns=APP.config.get('USER_NAMESPACE', False), ) SESSION.commit() - return flask.redirect(flask.url_for( - 'wait_task', taskid=taskid)) + return pagure.wait_for_task(taskid) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 811ff0c..97ead5e 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -29,6 +29,7 @@ import pagure.doc_utils import pagure.exceptions import pagure.lib import pagure.lib.git +import pagure.lib.tasks import pagure.forms from pagure import (APP, SESSION, login_required, __get_file_in_tree) @@ -891,8 +892,7 @@ def fork_project(repo, username=None, namespace=None): user=flask.g.fas_user.username) SESSION.commit() - return flask.redirect(flask.url_for( - 'wait_task', taskid=taskid)) + return pagure.wait_for_task(taskid) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover @@ -1085,11 +1085,30 @@ def new_remote_request_pull(repo, username=None, namespace=None): form = pagure.forms.RemoteRequestPullForm() if form.validate_on_submit(): + taskid = flask.request.values.get('taskid') + if taskid: + result = pagure.lib.tasks.get_result(taskid) + if not result.ready: + return pagure.wait_for_task_post( + taskid, form, 'new_remote_request_pull', + repo=repo.name, username=username, namespace=namespace) + # Make sure to collect any exceptions resulting from the task + result.get(timeout=0) + branch_from = form.branch_from.data.strip() branch_to = form.branch_to.data.strip() remote_git = form.git_repo.data.strip() repopath = pagure.get_remote_repo_path(remote_git, branch_from) + if not repopath: + taskid = pagure.lib.tasks.pull_remote_repo.delay( + repo.name, repo.namespace, repo.user.username, remote_git, + branch_from, branch_to) + return pagure.wait_for_task_post( + taskid, form, 'new_remote_request_pull', + repo=repo.name, username=username, namespace=namespace, + initial=True) + repo_obj = pygit2.Repository(repopath) try: @@ -1250,8 +1269,7 @@ def fork_edit_file( editfile=filename) SESSION.commit() - return flask.redirect(flask.url_for( - 'wait_task', taskid=taskid)) + return pagure.wait_for_task(taskid) except pagure.exceptions.PagureException as err: flask.flash(str(err), 'error') except SQLAlchemyError as err: # pragma: no cover diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index c464a1c..42aef77 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -2195,8 +2195,7 @@ def delete_branch(repo, branchname, username=None, namespace=None): taskid = pagure.lib.tasks.delete_branch.delay(repo, namespace, username, branchname).id - return flask.redirect(flask.url_for( - 'wait_task', taskid=taskid)) + return pagure.wait_for_task(taskid) @APP.route('/docs//') From c6f63a7594110db75b8e99521b6803cc02b35486 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 27/57] Taskify PR merge Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index 6d563f1..e9f21ab 100755 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -21,6 +21,7 @@ if 'PAGURE_CONFIG' not in os.environ \ import pagure # noqa: E402 import pagure.exceptions # noqa: E402 import pagure.lib.link # noqa: E402 +import pagure.lib.tasks # noqa: E402 from pagure.lib import REDIS # noqa: E402 @@ -93,15 +94,8 @@ def run_as_post_receive_hook(): print('/!\ Commit notification emails will not be sent and ' 'commits won\'t be logged') - try: - # Reset the merge_status of all opened PR to refresh their cache - pagure.lib.reset_status_pull_request(pagure.SESSION, project) - pagure.SESSION.commit() - except SQLAlchemyError as err: # pragma: no cover - pagure.SESSION.rollback() - print(err) - print('An error occured while running the default hook, please ' - 'report it to an admin.') + # Schedule refresh of all opened PRs + pagure.lib.tasks.refresh_pr_cache.delay(project.name, namespace, username) pagure.SESSION.remove() diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 18d28eb..a695ce1 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -1413,17 +1413,6 @@ def merge_pull_request( session, request, username, requestfolder=request_folder, ) - try: - # Reset the merge_status of all opened PR to refresh their cache - _log.info(' Clear the cached merged status of the other PRs') - pagure.lib.reset_status_pull_request(session, request.project) - session.commit() - except SQLAlchemyError as err: # pragma: no cover - session.rollback() - pagure.APP.logger.exception(err) - shutil.rmtree(newpath) - raise pagure.exceptions.PagureException( - 'Could not update this pull-request in the database') shutil.rmtree(newpath) return 'Changes merged!' diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 491e338..7e449d0 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -307,3 +307,31 @@ def pull_remote_repo(name, namespace, user, remote_git, branch_from, remote_git, clonepath, checkout_branch=branch_from) session.remove() + + +@conn.task +def refresh_pr_cache(name, namespace, user): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user) + + pagure.lib.reset_status_pull_request(session, project) + + session.remove() + +@conn.task +def merge_pull_request(name, namespace, user, requestid, user_merger): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user, with_lock=True) + request = pagure.lib.search_pull_requests( + session, project_id=project.id, requestid=requestid) + + pagure.lib.git.merge_pull_request( + session, request, user_merger, APP.config['REQUESTS_FOLDER']) + + refresh_pr_cache.delay(name, namespace, user) + session.remove() + return ret('view_repo', repo=name, username=user, namespace=namespace) diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 97ead5e..9f3614c 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -724,11 +724,10 @@ def merge_request_pull(repo, requestid, username=None, namespace=None): _log.info('All checks in the controller passed') try: - _log.info('Calling pagure.lib.git.merge_pull_request') - message = pagure.lib.git.merge_pull_request( - SESSION, request, flask.g.fas_user.username, - APP.config['REQUESTS_FOLDER']) - flask.flash(message) + taskid = pagure.lib.tasks.merge_pull_request.delay( + repo.name, namespace, username, requestid, + flask.g.fas_user.username) + return pagure.wait_for_task(taskid) except pygit2.GitError as err: _log.info('GitError exception raised') flask.flash(str(err.message), 'error') From cbb4a33b1282349f1d5b9fb75604a82513f4ed32 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 28/57] Add attachments volume Signed-off-by: Patrick Uiterwijk --- diff --git a/docker-compose.yml b/docker-compose.yml index aa5ee9e..c2cabbd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.2' volumes: repos: + attachments: services: web: build: @@ -17,6 +18,10 @@ services: source: repos target: /repos read_only: true + - type: volume + source: attachments + target: /attachments + read_only: false - .:/code worker: build: @@ -31,6 +36,10 @@ services: source: repos target: /repos read_only: false + - type: volume + source: attachments + target: /attachments + read_only: true - .:/code environment: - PYTHONPATH=. diff --git a/docker/web b/docker/web index 0d434e1..3f1d9e1 100644 --- a/docker/web +++ b/docker/web @@ -23,3 +23,4 @@ EXPOSE 5000 # Code injection is last to make optimal use of caches VOLUME ["/code"] # Openshift: COPY / /code +VOLUME ["/attachments"] diff --git a/docker/worker b/docker/worker index 98cd896..18072d3 100644 --- a/docker/worker +++ b/docker/worker @@ -21,5 +21,6 @@ ENTRYPOINT ["/usr/bin/celery", "-A", "pagure.lib.tasks", "worker", "--loglevel", # Code injection is last to make optimal use of caches VOLUME ["/code"] # Openshift: COPY / /code +VOLUME ["/attachments"] # Ideally this would run as non-root, but that needs the /repos owned correctly ENV C_FORCE_ROOT true diff --git a/openshift.cfg b/openshift.cfg index 8cdd73b..94e6544 100644 --- a/openshift.cfg +++ b/openshift.cfg @@ -9,3 +9,4 @@ DOCS_FOLDER = '/repos/docs' TICKETS_FOLDER = '/repos/tickets' REQUESTS_FOLDER = '/repos/requests' REMOTE_GIT_FOLDER = '/repos/remote' +ATTACHMENTS_FOLDER = '/attachments' diff --git a/pagure/default_config.py b/pagure/default_config.py index d4e3dc5..56bae2d 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -126,6 +126,13 @@ REMOTE_GIT_FOLDER = os.path.join( 'remotes' ) +# Folder containing attachments +ATTACHMENTS_FOLDER = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + '..', + 'attachments' +) + # Whether to enable scanning for viruses in attachments VIRUS_SCAN_ATTACHMENTS = False From 96c928ba79d0d0ca851a79475f2cc95be1ca7054 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 29/57] Use attachments folder as local cache asn taskify adding attachments to git Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/issue.py b/pagure/api/issue.py index 140dc88..c8eccd7 100644 --- a/pagure/api/issue.py +++ b/pagure/api/issue.py @@ -227,10 +227,10 @@ def api_new_issue(repo, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 8f4d0d9..df3a339 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -22,6 +22,7 @@ except ImportError: # pragma: no cover import json import datetime +import hashlib import logging import markdown import os @@ -2493,6 +2494,39 @@ def reset_status_pull_request(session, project): session.commit() +def add_attachment(repo, issue, attachmentfolder, user, filename, filestream): + ''' Add a file to the attachments folder of repo and update git. ''' + _log.info( + 'Addinf file: %s to the git repo: %s', + repo.path, werkzeug.secure_filename(filename)) + + # Prefix the filename with a timestamp: + filename = '%s-%s' % ( + hashlib.sha256(filestream.read()).hexdigest(), + werkzeug.secure_filename(filename) + ) + filedir = os.path.join(attachmentfolder, repo.fullname) + filepath = os.path.join(filedir, filename) + + if os.path.exists(filepath): + return filename + + if not os.path.exists(filedir): + os.makedirs(filedir) + + # Write file + filestream.seek(0) + with open(filepath, 'w') as stream: + stream.write(filestream.read()) + + tasks.add_file_to_git.delay( + repo.name, repo.namespace, + repo.user.username if repo.is_fork else None, + user, issue.uid, filename) + + return filename + + def get_issue_statuses(session): ''' Return the complete list of status an issue can have. ''' @@ -2670,7 +2704,6 @@ def avatar_url_from_email(email, size=64, default='retro', dns=False): ) else: import urllib - import hashlib query = urllib.urlencode({'s': size, 'd': default}) hashhex = hashlib.sha256(email).hexdigest() return "https://seccdn.libravatar.org/avatar/%s?%s" % ( diff --git a/pagure/lib/git.py b/pagure/lib/git.py index a695ce1..18a8522 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -826,30 +826,19 @@ def update_request_from_git( session.commit() -def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): +def _add_file_to_git(repo, issue, attachmentfolder, ticketfolder, user, + filename): ''' Add a given file to the specified ticket git repository. :arg repo: the Project object from the database + :arg attachmentfolder: the folder on the filesystem where the attachments + are stored :arg ticketfolder: the folder on the filesystem where the git repo for tickets are stored :arg user: the user object with its username and email :arg filename: the name of the file to save - :arg filestream: the actual content of the file ''' - _log.info( - 'Addinf file: %s to the git repo: %s', - repo.path, werkzeug.secure_filename(filename)) - - if not ticketfolder: - return - - # Prefix the filename with a timestamp: - filename = '%s-%s' % ( - hashlib.sha256(filestream.read()).hexdigest(), - werkzeug.secure_filename(filename) - ) - # Get the fork repopath = os.path.join(ticketfolder, repo.path) @@ -864,10 +853,7 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): index = new_repo.index # Are we adding files - added = False - if not os.path.exists(file_path): - added = True - else: + if os.path.exists(file_path): # File exists, remove the clone and return shutil.rmtree(newpath) return os.path.join('files', filename) @@ -875,26 +861,19 @@ def add_file_to_git(repo, issue, ticketfolder, user, filename, filestream): if not os.path.exists(folder_path): os.mkdir(folder_path) - # Write down what changed - filestream.seek(0) - with open(file_path, 'w') as stream: - stream.write(filestream.read()) + # Copy from attachments directory + src = os.path.join(attachmentfolder, repo.fullname, filename) + shutil.copyfile(src, file_path) # Retrieve the list of files that changed diff = new_repo.diff() files = [patch.new_file_path for patch in diff] # Add the changes to the index - if added: - index.add(os.path.join('files', filename)) + index.add(os.path.join('files', filename)) for filename in files: index.add(filename) - # If not change, return - if not files and not added: - shutil.rmtree(newpath) - return - # See if there is a parent to this commit parent = None try: diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 7e449d0..e035ad1 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -19,11 +19,15 @@ import pygit2 import tempfile import six +import logging + import pagure from pagure import APP import pagure.lib import pagure.lib.git +_log = logging.getLogger(__name__) + conn = Celery('tasks', broker='redis://%s' % APP.config['REDIS_HOST'], @@ -335,3 +339,18 @@ def merge_pull_request(name, namespace, user, requestid, user_merger): refresh_pr_cache.delay(name, namespace, user) session.remove() return ret('view_repo', repo=name, username=user, namespace=namespace) + + +@conn.task +def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): + session = pagure.lib.create_session() + + project = pagure.lib._get_project(session, namespace=namespace, + name=name, user=user) + issue = pagure.lib.get_issue_by_uid(session, issueuid) + + pagure.lib.git._add_file_to_git( + project, issue, APP.config['ATTACHMENTS_FOLDER'], + APP.config['TICKETS_FOLDER'], user_attacher, filename) + + session.remove() diff --git a/pagure/ui/issues.py b/pagure/ui/issues.py index 7d1687c..9c897b4 100644 --- a/pagure/ui/issues.py +++ b/pagure/ui/issues.py @@ -904,10 +904,10 @@ def new_issue(repo, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1164,10 +1164,10 @@ def edit_issue(repo, issueid, username=None, namespace=None): # If there is a file attached, attach it. filestream = flask.request.files.get('filestream') if filestream and '' in issue.content: - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1250,10 +1250,10 @@ def upload_issue(repo, issueid, username=None, namespace=None): if form.validate_on_submit(): filestream = flask.request.files['filestream'] - new_filename = pagure.lib.git.add_file_to_git( + new_filename = pagure.lib.add_attachment( repo=repo, issue=issue, - ticketfolder=APP.config['TICKETS_FOLDER'], + attachmentfolder=APP.config['ATTACHMENTS_FOLDER'], user=user_obj, filename=filestream.filename, filestream=filestream.stream, @@ -1291,33 +1291,48 @@ def view_issue_raw_file( if not repo.settings.get('issue_tracker', True): flask.abort(404, 'No issue tracker found for this project') - reponame = os.path.join(APP.config['TICKETS_FOLDER'], repo.path) + mimetype, encoding = mimetypes.guess_type(filename) - repo_obj = pygit2.Repository(reponame) + attachdir = os.path.join(APP.config['ATTACHMENTS_FOLDER'], repo.fullname) + attachpath = os.path.join(attachdir, filename) + if not os.path.exists(attachpath): + if not os.path.exists(attachdir): + os.makedirs(attachdir) - if repo_obj.is_empty: - flask.abort(404, 'Empty repo cannot have a file') + # Try to copy from git repo to attachments folder + reponame = os.path.join(APP.config['TICKETS_FOLDER'], repo.path) + repo_obj = pygit2.Repository(reponame) - branch = repo_obj.lookup_branch('master') - commit = branch.get_object() + if repo_obj.is_empty: + flask.abort(404, 'Empty repo cannot have a file') - mimetype = None - encoding = None + branch = repo_obj.lookup_branch('master') + commit = branch.get_object() - content = __get_file_in_tree( - repo_obj, commit.tree, filename.split('/'), bail_on_tree=True) - if not content or isinstance(content, pygit2.Tree): - flask.abort(404, 'File not found') + content = __get_file_in_tree( + repo_obj, commit.tree, ['files', filename], bail_on_tree=True) + if not content or isinstance(content, pygit2.Tree): + flask.abort(404, 'File not found') - mimetype, encoding = mimetypes.guess_type(filename) - data = repo_obj[content.oid].data + data = repo_obj[content.oid].data + + if not data: + flask.abort(404, 'No content found') + + _log.info("Migrating file %s for project %s to attachments", + filename, repo.fullname) + + with open(attachpath, 'w') as stream: + stream.write(data) + data = None - if not data: - flask.abort(404, 'No content found') + # At this moment, attachpath exists and points to the file + with open(attachpath, 'r') as f: + data = f.read() if not raw \ and (filename.endswith('.patch') or filename.endswith('.diff')) \ - and not is_binary_string(content.data): + and not is_binary_string(data): # We have a patch file attached to this issue, render the diff in html orig_filename = filename.partition('-')[2] return flask.render_template( From f7ddd205a2b0c4b39ab306b7316f5da356ad4edb Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 30/57] Add default plugin for new projects Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index e035ad1..01da2f0 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -126,16 +126,15 @@ def create_project(username, namespace, name, add_readme, ignore_existing_repo): mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP) # Install the default hook - # TODO: This needs enabling, but doesn't work in my env due to tables - #plugin = pagure.lib.plugins.get_plugin('default') - #dbobj = plugin.db_object() - #dbobj.active = True - #dbobj.project_id = project.id - #session.add(dbobj) - #session.flush() - #plugin.set_up(project) - #plugin.install(project, dbobj) - #session.commit() + plugin = pagure.lib.plugins.get_plugin('default') + dbobj = plugin.db_object() + dbobj.active = True + dbobj.project_id = project.id + session.add(dbobj) + session.flush() + plugin.set_up(project) + plugin.install(project, dbobj) + session.commit() session.remove() generate_gitolite_acls.delay() From d0ace20c18ddfcff20079fc7a087a800d0ca8488 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 31/57] Default hook does not need a lock anymore Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index e9f21ab..b51b77a 100755 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -40,8 +40,7 @@ def run_as_post_receive_hook(): print('namespace:', namespace) project = pagure.lib._get_project( - pagure.SESSION, repo, user=username, namespace=namespace, - with_lock=True) + pagure.SESSION, repo, user=username, namespace=namespace) for line in sys.stdin: if pagure.APP.config.get('HOOK_DEBUG', False): From 539a7207662c56402cd01332317279728f714198 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 32/57] Make the tests run tasks eagerly Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/default_config.py b/pagure/default_config.py index 56bae2d..3e8f546 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -83,6 +83,9 @@ MAX_CONTENT_LENGTH = 4 * 1024 * 1024 # 4 megabytes # IP addresses allowed to access the internal endpoints IP_ALLOWED_INTERNAL = ['127.0.0.1', 'localhost', '::1'] +# Worker configuration +CELERY_CONFIG = {} + # Redis configuration EVENTSOURCE_SOURCE = None WEBHOOK = False diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 01da2f0..cbdf53d 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -32,6 +32,7 @@ _log = logging.getLogger(__name__) conn = Celery('tasks', broker='redis://%s' % APP.config['REDIS_HOST'], backend='redis://%s' % APP.config['REDIS_HOST']) +conn.conf.update(APP.config['CELERY_CONFIG']) def get_result(uuid): diff --git a/tests/test_config b/tests/test_config index d52090c..f5f56dc 100644 --- a/tests/test_config +++ b/tests/test_config @@ -1 +1,3 @@ PAGURE_CI_SERVICES = ['jenkins'] +# Execute tasks without worker +CELERY_CONFIG = {'CELERY_ALWAYS_EAGER': True} From 3caaa0538c0816305309829e27ec6b03ecbe0dd2 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 33/57] Flake8 fixes Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/hooks/files/default_hook.py b/pagure/hooks/files/default_hook.py index b51b77a..ab73492 100755 --- a/pagure/hooks/files/default_hook.py +++ b/pagure/hooks/files/default_hook.py @@ -11,8 +11,6 @@ import sys import pygit2 -from sqlalchemy.exc import SQLAlchemyError - if 'PAGURE_CONFIG' not in os.environ \ and os.path.exists('/etc/pagure/pagure.cfg'): os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index df3a339..51e8984 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -26,7 +26,6 @@ import hashlib import logging import markdown import os -import shutil import tempfile import subprocess import urlparse @@ -44,8 +43,6 @@ from sqlalchemy.orm import aliased from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import scoped_session -import pygit2 - import pagure import pagure.exceptions import pagure.lib.git @@ -91,6 +88,8 @@ def get_user(session, key): SESSIONMAKER = None + + def create_session(db_url=None, debug=False, pool_recycle=3600): ''' Create the Session object to use to query the database. @@ -118,7 +117,7 @@ def create_session(db_url=None, debug=False, pool_recycle=3600): SESSIONMAKER = sessionmaker(bind=engine) scopedsession = scoped_session(SESSIONMAKER) - # model.BASE.metadata.bind = scopedsession + model.BASE.metadata.bind = scopedsession return scopedsession @@ -1729,7 +1728,7 @@ def update_user_settings(session, settings, user): update.append(key) new_settings[key] = settings[key] else: - if new_settings[key] != False: + if new_settings[key] is not False: update.append(key) new_settings[key] = False @@ -1747,7 +1746,6 @@ def fork_project(session, user, repo, gitfolder, docfolder, ticketfolder, requestfolder, editbranch=None, editfile=None): ''' Fork a given project into the user's forks. ''' - reponame = os.path.join(gitfolder, repo.path) forkreponame = '%s.git' % os.path.join( gitfolder, 'forks', user, repo.namespace if repo.namespace else '', repo.name) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 18a8522..2a9abe2 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -15,7 +15,6 @@ # pylint: disable=too-many-lines import datetime -import hashlib import json import logging import os @@ -207,7 +206,6 @@ def _generate_gitolite_acls(): def update_git(obj, repo, repofolder): """ Schedules an update_repo task after determining arguments. """ - user = None ticketuid = None requestuid = None if obj.isa == 'issue': diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index cbdf53d..59ac0c4 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -51,7 +51,8 @@ def generate_gitolite_acls(): @conn.task -def create_project(username, namespace, name, add_readme, ignore_existing_repo): +def create_project(username, namespace, name, add_readme, + ignore_existing_repo): session = pagure.lib.create_session() project = pagure.lib._get_project(session, namespace=namespace, @@ -302,9 +303,6 @@ def pull_remote_repo(name, namespace, user, remote_git, branch_from, branch_to): session = pagure.lib.create_session() - project = pagure.lib._get_project(session, namespace=namespace, - name=name, user=user) - clonepath = pagure.get_remote_repo_path(remote_git, branch_from, ignore_non_exist=True) pygit2.clone_repository( @@ -324,6 +322,7 @@ def refresh_pr_cache(name, namespace, user): session.remove() + @conn.task def merge_pull_request(name, namespace, user, requestid, user_merger): session = pagure.lib.create_session() diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 84c98cc..a771613 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -501,6 +501,7 @@ def wait_task(taskid): 'waiting.html', taskid=taskid) + @APP.route('/settings/', methods=('GET', 'POST')) @APP.route('/settings', methods=('GET', 'POST')) @login_required From 226add163da5366c309ff7142db2e67eae95a1e7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 34/57] Fix pull_remote_repo Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 59ac0c4..ad4f568 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -299,16 +299,13 @@ def fork(name, namespace, user_owner, user_forker, editbranch, editfile): @conn.task -def pull_remote_repo(name, namespace, user, remote_git, branch_from, - branch_to): - session = pagure.lib.create_session() - +def pull_remote_repo(remote_git, branch_from): clonepath = pagure.get_remote_repo_path(remote_git, branch_from, ignore_non_exist=True) pygit2.clone_repository( remote_git, clonepath, checkout_branch=branch_from) - session.remove() + return clonepath @conn.task diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index 9f3614c..def7cb2 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -1101,8 +1101,7 @@ def new_remote_request_pull(repo, username=None, namespace=None): repopath = pagure.get_remote_repo_path(remote_git, branch_from) if not repopath: taskid = pagure.lib.tasks.pull_remote_repo.delay( - repo.name, repo.namespace, repo.user.username, remote_git, - branch_from, branch_to) + remote_git, branch_from) return pagure.wait_for_task_post( taskid, form, 'new_remote_request_pull', repo=repo.name, username=username, namespace=namespace, From bb08fbc72b59292b46d3d619371fac339a10c1ea Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 35/57] Make new project and forking a project API async Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/project.py b/pagure/api/project.py index c8e7fd5..5dcb7f9 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -408,6 +408,8 @@ def api_new_project(): -------------------- Create a new project on this pagure instance. + This is an asynchronous call. + :: POST /api/0//new @@ -455,7 +457,8 @@ def api_new_project(): :: { - 'message': 'Project "foo" created' + 'message': 'Project creation queued', + 'taskid': '123-abcd' } """ @@ -488,7 +491,7 @@ def api_new_project(): private = form.private.data try: - message = pagure.lib.new_project( + taskid = pagure.lib.new_project( SESSION, name=name, namespace=namespace, @@ -511,7 +514,8 @@ def api_new_project(): ) SESSION.commit() pagure.lib.git.generate_gitolite_acls() - output['message'] = message + output = {'message': 'Project creation queued', + 'taskid': taskid} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) @@ -537,6 +541,8 @@ def api_fork_project(): -------------------- Fork a project on this pagure instance. + This is an asynchronous call. + :: POST /api/0//fork @@ -565,7 +571,8 @@ def api_fork_project(): :: { - "message": 'Repo "test" cloned to "pingou/test"' + "message": "Project forking queued", + "taskid": "123-abcd" } """ @@ -584,7 +591,7 @@ def api_fork_project(): 404, error_code=APIERROR.ENOPROJECT) try: - message = pagure.lib.fork_project( + taskid = pagure.lib.fork_project( SESSION, user=flask.g.fas_user.username, repo=repo, @@ -594,8 +601,8 @@ def api_fork_project(): requestfolder=APP.config['REQUESTS_FOLDER'], ) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() - output['message'] = message + output = {'message': 'Project forking queued', + 'taskid': taskid} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) From 7d254740dff92175dffd063697443847d1a9fb8a Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 36/57] Remove some tests Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/test_pagure_flask.py b/tests/test_pagure_flask.py index 759d4b6..b2358c5 100644 --- a/tests/test_pagure_flask.py +++ b/tests/test_pagure_flask.py @@ -41,40 +41,14 @@ class PagureGetRemoteRepoPath(tests.Modeltests): tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) tests.add_content_git_repo(os.path.join(self.path, 'repos', 'test2.git')) - def test_failed_clone(self): - """ Test get_remote_repo_path in pagure. """ - with self.assertRaises(werkzeug.exceptions.InternalServerError) as cm: - pagure.get_remote_repo_path('remote_repo', 'branch') - - self.assertEqual( - cm.exception.get_description(), - '

The following error was raised when trying to clone the ' - 'remote repo: Unsupported URL protocol

') - - @mock.patch( - 'pagure.lib.repo.PagureRepo.pull', - mock.MagicMock(side_effect=pygit2.GitError)) - def test_failed_pull(self): - """ Test get_remote_repo_path in pagure. """ - pagure.get_remote_repo_path( - os.path.join(self.path, 'repos', 'test2.git'), 'master') - - with self.assertRaises(werkzeug.exceptions.InternalServerError) as cm: - pagure.get_remote_repo_path( - os.path.join(self.path, 'repos', 'test2.git'), 'master') - - self.assertEqual( - cm.exception.get_description(), - '

The following error was raised when trying to pull the ' - 'changes from the remote:

') - @mock.patch( 'pagure.lib.repo.PagureRepo.pull', mock.MagicMock(side_effect=pygit2.GitError)) def test_passing(self): """ Test get_remote_repo_path in pagure. """ output = pagure.get_remote_repo_path( - os.path.join(self.path, 'repos', 'test2.git'), 'master') + os.path.join(self.path, 'repos', 'test2.git'), 'master', + ignore_non_exist=True) self.assertTrue(output.endswith('repos_test2.git_master')) From e6f1fdf673f0b7fb5e0959742d7bab83050cce00 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 37/57] Make fork API async Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/fork.py b/pagure/api/fork.py index 7457fcb..9621731 100644 --- a/pagure/api/fork.py +++ b/pagure/api/fork.py @@ -15,6 +15,7 @@ from sqlalchemy.exc import SQLAlchemyError import pagure import pagure.exceptions import pagure.lib +import pagure.lib.tasks from pagure import APP, SESSION, is_repo_committer from pagure.api import (API, api_method, api_login_required, APIERROR, get_authorized_api_project) @@ -287,6 +288,8 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): -------------------- Instruct Paugre to merge a pull request. + This is an asynchronous call. + :: POST /api/0//pull-request//merge @@ -303,7 +306,8 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): :: { - "message": "Changes merged!" + "message": "Merging queued", + "taskid": "123-abcd" } """ # noqa @@ -345,10 +349,11 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): raise pagure.exceptions.APIError(403, error_code=APIERROR.EPRSCORE) try: - message = pagure.lib.git.merge_pull_request( - SESSION, request, flask.g.fas_user.username, - APP.config['REQUESTS_FOLDER']) - output['message'] = message + taskid = pagure.lib.tasks.merge_pull_request.delay( + repo.name, namespace, username, requestid, + flask.g.fas_user.username) + output = {'message': 'Merging queued', + 'taskid': taskid} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) From 61fdf21ee5640890bae6ac28070e8858cc02a866 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 38/57] Add task status API Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/__init__.py b/pagure/api/__init__.py index f600bc5..def1205 100644 --- a/pagure/api/__init__.py +++ b/pagure/api/__init__.py @@ -28,6 +28,7 @@ API = flask.Blueprint('api_ns', __name__, url_prefix='/api/0') import pagure # noqa: E402 import pagure.lib # noqa: E402 +import pagure.lib.tasks from pagure import __api_version__, APP, SESSION, authenticated # noqa: E402 from pagure.doc_utils import load_doc, modify_rst, modify_html # noqa: E402 from pagure.exceptions import APIError # noqa: E402 @@ -312,6 +313,24 @@ def api_users(): ) +@API.route('/task//status') +@API.route('/task//status/') +def api_task_status(taskid): + ''' + Return the status of a async task + ''' + result = pagure.lib.tasks.get_result(taskid) + if not result.ready: + output = {'ready': False, + 'status': result.status} + else: + output = {'ready': True, + 'succesful': result.succesful(), + 'status': result.status} + + return flask.jsonify(output) + + @API.route('//tags') @API.route('//tags/') @API.route('/fork///tags') diff --git a/pagure/api/project.py b/pagure/api/project.py index 5dcb7f9..b2aaf2d 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -513,7 +513,6 @@ def api_new_project(): user_ns=APP.config.get('USER_NAMESPACE', False), ) SESSION.commit() - pagure.lib.git.generate_gitolite_acls() output = {'message': 'Project creation queued', 'taskid': taskid} except pagure.exceptions.PagureException as err: From 55e7e06a6fca003de410b3191ef3ca9a4c2723a8 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 39/57] Make test suite start a broker and worker Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index ad4f568..26dd747 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -29,9 +29,12 @@ import pagure.lib.git _log = logging.getLogger(__name__) -conn = Celery('tasks', - broker='redis://%s' % APP.config['REDIS_HOST'], - backend='redis://%s' % APP.config['REDIS_HOST']) +if os.environ.get('PAGURE_BROKER_URL'): + broker_url = os.environ['PAGURE_BROKER_URL'] +else: + broker_url = 'redis://%s' % APP.config['REDIS_HOST'] + +conn = Celery('tasks', broker=broker_url, backend=broker_url) conn.conf.update(APP.config['CELERY_CONFIG']) diff --git a/tests/__init__.py b/tests/__init__.py index f43b640..2cbcf34 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -14,6 +14,7 @@ import pkg_resources import logging import unittest import shutil +import subprocess import sys import tempfile import os @@ -43,7 +44,7 @@ import pagure.lib.model from pagure.lib.repo import PagureRepo import pagure.perfrepo as perfrepo -DB_PATH = 'sqlite:///:memory:' +DB_PATH = None FAITOUT_URL = 'http://faitout.fedorainfracloud.org/' if os.environ.get('FAITOUT_URL'): FAITOUT_URL = os.environ.get('FAITOUT_URL') @@ -55,6 +56,17 @@ PAGLOG = logging.getLogger('pagure') PAGLOG.setLevel(logging.CRITICAL) PAGLOG.handlers = [] +CONFIG_TEMPLATE = """ +PAGURE_CI_SERVICES = ['jenkins'] +GIT_FOLDER = '%(path)s/repos' +DOCS_FOLDER = '%(path)s/docs' +TICKETS_FOLDER = '%(path)s/tickets' +REQUESTS_FOLDER = '%(path)s/requests' +REMOTE_GIT_FOLDER = '%(path)s/remotes' +ATTACHMENTS_FOLDER = '%(path)s/attachments' +DB_URL = '%(dburl)s' +""" + LOG.info('BUILD_ID: %s', os.environ.get('BUILD_ID')) if os.environ.get('BUILD_ID')or os.environ.get('FAITOUT_URL'): @@ -133,16 +145,60 @@ class Modeltests(unittest.TestCase): perfrepo.reset_stats() perfrepo.REQUESTS = [] - # Clean up eventual git repo left in the present folder. pagure.REDIS = None pagure.lib.REDIS = None - self.path = tempfile.mkdtemp(prefix='pagure-tests') + if self.path is not None: + raise Exception('Double init?!') + self.path = tempfile.mkdtemp(prefix='pagure-tests-path-') + LOG.info('Testdir: %s', self.path) for folder in ['tickets', 'repos', 'forks', 'docs', 'requests', 'releases', 'remotes']: os.mkdir(os.path.join(self.path, folder)) + if DB_PATH: + self.dbpath = DB_PATH + else: + self.dbpath = 'sqlite:///%s' % os.path.join(self.path, + 'db.sqlite') + + # Write a config file + config_values = {'path': self.path, + 'dburl': self.dbpath} + with open(os.path.join(self.path, 'config'), 'w') as f: + f.write(CONFIG_TEMPLATE % config_values) + + # Create a broker + broker_url = os.path.join(self.path, 'broker') + + self.broker = subprocess.Popen( + ['/usr/bin/redis-server', '--unixsocket', broker_url, '--port', + '0', '--loglevel', 'warning']) + self.broker.poll() + if self.broker.returncode is not None: + raise Exception('Broker failed to start') + self.session = pagure.lib.model.create_tables( - DB_PATH, acls=pagure.APP.config.get('ACLS', {})) + self.dbpath, acls=pagure.APP.config.get('ACLS', {})) + + reload(pagure.lib.tasks) + celery_broker_url = 'redis+socket://' + broker_url + pagure.lib.tasks.conn.conf.broker_url = celery_broker_url + pagure.lib.tasks.conn.conf.result_backend = celery_broker_url + + # Start a worker + # Using cocurrency 2 to test with some concurrency, but not be heavy + # Using eventlet so that worker.terminate kills everything + self.worker = subprocess.Popen( + ['/usr/bin/celery', '-A', 'pagure.lib.tasks', 'worker', + '--loglevel', 'info', '--concurrency', '2', '--pool', 'eventlet'], + env={'PAGURE_BROKER_URL': celery_broker_url, + 'PAGURE_CONFIG': os.path.join(self.path, 'config'), + 'PYTHONPATH': '.'}, + cwd=os.path.normpath(os.path.join(os.path.dirname(__file__), + '..'))) + self.worker.poll() + if self.worker.returncode is not None: + raise Exception('Worker failed to start') # Create a couple of users item = pagure.lib.model.User( @@ -191,17 +247,20 @@ class Modeltests(unittest.TestCase): """ Remove the test.db database if there is one. """ self.session.close() - # Clear temp directory - shutil.rmtree(self.path) - # Clear DB - if os.path.exists(DB_PATH): - os.unlink(DB_PATH) - if DB_PATH.startswith('postgres'): - if 'localhost' not in DB_PATH: - db_name = DB_PATH.rsplit('/', 1)[1] + if self.dbpath.startswith('postgres'): + if 'localhost' not in self.dbpath: + db_name = self.dbpath.rsplit('/', 1)[1] requests.get('%s/clean/%s' % (FAITOUT_URL, db_name)) + # Terminate worker and broker + self.worker.terminate() + self.broker.terminate() + + # Remove testdir + shutil.rmtree(self.path) + self.path = None + def get_csrf(self, url='/new'): """Retrieve a CSRF token from given URL.""" output = self.app.get(url) diff --git a/tests/test_config b/tests/test_config index f5f56dc..d52090c 100644 --- a/tests/test_config +++ b/tests/test_config @@ -1,3 +1 @@ PAGURE_CI_SERVICES = ['jenkins'] -# Execute tasks without worker -CELERY_CONFIG = {'CELERY_ALWAYS_EAGER': True} From 0281fff14e3f9bec9d1e7f2a77d4f3747ba31fda Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:36:59 +0000 Subject: [PATCH 40/57] Make async optional in project and fork API Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/fork.py b/pagure/api/fork.py index 9621731..ebd01f3 100644 --- a/pagure/api/fork.py +++ b/pagure/api/fork.py @@ -305,11 +305,17 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): :: + wait=False: { "message": "Merging queued", "taskid": "123-abcd" } + wait=True: + { + "message": "Changes merged!" + } + """ # noqa output = {} @@ -351,9 +357,13 @@ def api_pull_request_merge(repo, requestid, username=None, namespace=None): try: taskid = pagure.lib.tasks.merge_pull_request.delay( repo.name, namespace, username, requestid, - flask.g.fas_user.username) + flask.g.fas_user.username).id output = {'message': 'Merging queued', 'taskid': taskid} + + if flask.request.form.get('wait', True): + pagure.lib.tasks.get_result(taskid).get() + output = {'message': 'Changes merged!'} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) diff --git a/pagure/api/project.py b/pagure/api/project.py index b2aaf2d..d39c4d9 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -450,17 +450,29 @@ def api_new_project(): | | | | projects, confirm this | | | | | with your administrators| +------------------+---------+--------------+---------------------------+ + | ``wait`` | boolean | Optional | | A boolean to specify if | + | | | | this API call should | + | | | | return a taskid or if it| + | | | | should wait for the task| + | | | | to finish. | + +------------------+---------+--------------+---------------------------+ Sample response ^^^^^^^^^^^^^^^ :: + wait=False: { 'message': 'Project creation queued', 'taskid': '123-abcd' } + wait=True: + { + 'message': 'Project creation queued' + } + """ user = pagure.lib.search_user(SESSION, username=flask.g.fas_user.username) output = {} @@ -515,6 +527,13 @@ def api_new_project(): SESSION.commit() output = {'message': 'Project creation queued', 'taskid': taskid} + + if flask.request.form.get('wait', True): + result = pagure.lib.tasks.get_result(taskid).get() + project = pagure.lib._get_project( + SESSION, name=result['repo'], + namespace=result['namespace']) + output = {'message': 'Project "%s" created' % project.fullname} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) @@ -562,6 +581,12 @@ def api_fork_project(): | ``username`` | string | Optional | | The username of the user| | | | | of the fork. | +------------------+---------+--------------+---------------------------+ + | ``wait`` | boolean | Optional | | A boolean to specify if | + | | | | this API call should | + | | | | return a taskid or if it| + | | | | should wait for the task| + | | | | to finish. | + +------------------+---------+--------------+---------------------------+ Sample response @@ -569,11 +594,17 @@ def api_fork_project(): :: + wait=False: { "message": "Project forking queued", "taskid": "123-abcd" } + wait=True: + { + "message": 'Repo "test" cloned to "pingou/test" + } + """ output = {} @@ -602,6 +633,12 @@ def api_fork_project(): SESSION.commit() output = {'message': 'Project forking queued', 'taskid': taskid} + + if flask.request.form.get('wait', True): + pagure.lib.tasks.get_result(taskid).get() + output = {'message': 'Repo "%s" cloned to "%s/%s"' + % (repo.fullname, flask.g.fas_user.username, + repo.fullname)} except pagure.exceptions.PagureException as err: raise pagure.exceptions.APIError( 400, error_code=APIERROR.ENOCODE, error=str(err)) From 02a7ccc7e94abe6c9d402fcf7b9e654e1dfc6f5d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 41/57] For PR test, make actual commits Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index 2cbcf34..b263f27 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -57,7 +57,6 @@ PAGLOG.setLevel(logging.CRITICAL) PAGLOG.handlers = [] CONFIG_TEMPLATE = """ -PAGURE_CI_SERVICES = ['jenkins'] GIT_FOLDER = '%(path)s/repos' DOCS_FOLDER = '%(path)s/docs' TICKETS_FOLDER = '%(path)s/tickets' @@ -258,7 +257,7 @@ class Modeltests(unittest.TestCase): self.broker.terminate() # Remove testdir - shutil.rmtree(self.path) + #shutil.rmtree(self.path) self.path = None def get_csrf(self, url='/new'): @@ -561,11 +560,12 @@ Dev instance: http://209.132.184.222/ (/!\\ May change unexpectedly, it's a dev shutil.rmtree(newfolder) -def add_commit_git_repo(folder, ncommits=10, filename='sources'): +def add_commit_git_repo(folder, ncommits=10, filename='sources', + branch='master'): """ Create some more commits for the specified git repo. """ if not os.path.exists(folder): os.makedirs(folder) - brepo = pygit2.init_repository(folder, bare=True) + pygit2.init_repository(folder, bare=True) newfolder = tempfile.mkdtemp(prefix='pagure-tests') repo = pygit2.clone_repository(folder, newfolder) @@ -593,7 +593,7 @@ def add_commit_git_repo(folder, ncommits=10, filename='sources'): committer = pygit2.Signature( 'Cecil Committer', 'cecil@committers.tld') repo.create_commit( - 'refs/heads/master', # the name of the reference to update + 'refs/heads/master', author, committer, 'Add row %s to %s file' % (index, filename), @@ -605,10 +605,7 @@ def add_commit_git_repo(folder, ncommits=10, filename='sources'): # Push to origin ori_remote = repo.remotes[0] - master_ref = repo.lookup_reference('HEAD').resolve() - refname = '%s:%s' % (master_ref.name, master_ref.name) - - PagureRepo.push(ori_remote, refname) + PagureRepo.push(ori_remote, 'HEAD:refs/heads/%s' % branch) shutil.rmtree(newfolder) diff --git a/tests/test_pagure_flask_api_fork.py b/tests/test_pagure_flask_api_fork.py index f088c38..dfcb704 100644 --- a/tests/test_pagure_flask_api_fork.py +++ b/tests/test_pagure_flask_api_fork.py @@ -483,13 +483,17 @@ class PagureFlaskApiForktests(tests.Modeltests): ) @patch('pagure.lib.notify.send_email') - @patch('pagure.lib.git.merge_pull_request') - def test_api_pull_request_merge(self, mpr, send_email): + def test_api_pull_request_merge(self, send_email): """ Test the api_pull_request_merge method of the flask api. """ - mpr.return_value = 'Changes merged!' send_email.return_value = True tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + tests.create_projects_git(os.path.join(self.path, 'requests'), + bare=True) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + tests.add_commit_git_repo(os.path.join(self.path, 'repos', 'test.git'), + branch='test') tests.create_tokens(self.session) tests.create_tokens_acl(self.session) @@ -499,7 +503,7 @@ class PagureFlaskApiForktests(tests.Modeltests): req = pagure.lib.new_pull_request( session=self.session, repo_from=forked_repo, - branch_from='master', + branch_from='test', repo_to=repo, branch_to='master', title='test pull-request', @@ -596,13 +600,17 @@ class PagureFlaskApiForktests(tests.Modeltests): ) @patch('pagure.lib.notify.send_email') - @patch('pagure.lib.git.merge_pull_request') - def test_api_pull_request_merge_user_token(self, mpr, send_email): + def test_api_pull_request_merge_user_token(self, send_email): """ Test the api_pull_request_merge method of the flask api. """ - mpr.return_value = 'Changes merged!' send_email.return_value = True tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + tests.create_projects_git(os.path.join(self.path, 'requests'), + bare=True) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + tests.add_commit_git_repo(os.path.join(self.path, 'repos', 'test.git'), + branch='test') tests.create_tokens(self.session, project_id=None) tests.create_tokens_acl(self.session) @@ -612,7 +620,7 @@ class PagureFlaskApiForktests(tests.Modeltests): req = pagure.lib.new_pull_request( session=self.session, repo_from=forked_repo, - branch_from='master', + branch_from='test', repo_to=repo, branch_to='master', title='test pull-request', diff --git a/tests/test_pagure_flask_api_ui_private_repo.py b/tests/test_pagure_flask_api_ui_private_repo.py index e952475..5f60edd 100644 --- a/tests/test_pagure_flask_api_ui_private_repo.py +++ b/tests/test_pagure_flask_api_ui_private_repo.py @@ -1811,10 +1811,8 @@ class PagurePrivateRepotest(tests.Modeltests): ) @patch('pagure.lib.notify.send_email') - @patch('pagure.lib.git.merge_pull_request') - def test_api_private_repo_pr_merge(self, mpr, send_email): + def test_api_private_repo_pr_merge(self, send_email): """ Test the api_pull_request_merge method of the flask api. """ - mpr.return_value = 'Changes merged!' send_email.return_value = True pagure.APP.config['REQUESTS_FOLDER'] = None @@ -1830,6 +1828,15 @@ class PagurePrivateRepotest(tests.Modeltests): self.session.add(item) self.session.commit() + tests.create_projects(self.session) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + tests.create_projects_git(os.path.join(self.path, 'requests'), + bare=True) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', + 'test4.git')) + tests.add_commit_git_repo(os.path.join(self.path, 'repos', + 'test4.git'), + branch='test') tests.create_tokens(self.session) tests.create_tokens_acl(self.session) @@ -1850,7 +1857,7 @@ class PagurePrivateRepotest(tests.Modeltests): req = pagure.lib.new_pull_request( session=self.session, repo_from=forked_repo, - branch_from='master', + branch_from='test', repo_to=repo, branch_to='master', title='test pull-request', From c00c89251717d83915402775a33666b0a99d8023 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 42/57] Make redis quiet Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index b263f27..70b11ae 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -171,7 +171,8 @@ class Modeltests(unittest.TestCase): self.broker = subprocess.Popen( ['/usr/bin/redis-server', '--unixsocket', broker_url, '--port', - '0', '--loglevel', 'warning']) + '0', '--loglevel', 'warning', '--logfile', '/dev/null'], + stdout=None, stderr=None) self.broker.poll() if self.broker.returncode is not None: raise Exception('Broker failed to start') @@ -189,7 +190,8 @@ class Modeltests(unittest.TestCase): # Using eventlet so that worker.terminate kills everything self.worker = subprocess.Popen( ['/usr/bin/celery', '-A', 'pagure.lib.tasks', 'worker', - '--loglevel', 'info', '--concurrency', '2', '--pool', 'eventlet'], + '--loglevel', 'info', '--concurrency', '2', '--pool', 'eventlet', + '--without-gossip', '--without-mingle', '--quiet'], env={'PAGURE_BROKER_URL': celery_broker_url, 'PAGURE_CONFIG': os.path.join(self.path, 'config'), 'PYTHONPATH': '.'}, From d30ca73306a6b94398c2b2c9743a262076e63361 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 43/57] Make worker terminate before killing broker Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index 70b11ae..fa181e8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -256,10 +256,12 @@ class Modeltests(unittest.TestCase): # Terminate worker and broker self.worker.terminate() + self.worker.wait() self.broker.terminate() + self.broker.wait() # Remove testdir - #shutil.rmtree(self.path) + shutil.rmtree(self.path) self.path = None def get_csrf(self, url='/new'): From b71da2c03e4c9b9c47c914dd084d1866c9d70caa Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 44/57] Wait for tasks in dump_load_tickets Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 51e8984..0aca725 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -1410,6 +1410,7 @@ def new_issue(session, repo, title, content, user, ticketfolder, issue_id=None, session.add(issue) # Make sure we won't have SQLAlchemy error before we create the issue session.flush() + session.commit() pagure.lib.git.update_git( issue, repo=repo, repofolder=ticketfolder) diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 2a9abe2..2a4ce2f 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -215,10 +215,20 @@ def update_git(obj, repo, repofolder): else: raise NotImplementedError('Unknown object type %s' % obj.isa) - return pagure.lib.tasks.update_git.delay( + queued = pagure.lib.tasks.update_git.delay( repo.name, repo.namespace, repo.user.username if repo.is_fork else None, ticketuid, requestuid) + _maybe_wait(queued) + return queued + + +def _maybe_wait(result): + """ Function to patch if one wants to wait for finish. + + This function should only ever be overridden by a few tests that depend on + counting and very precise timing. """ + pass def _update_git(obj, repo, repofolder): diff --git a/tests/test_pagure_flask_dump_load_ticket.py b/tests/test_pagure_flask_dump_load_ticket.py index 48ff558..83809b9 100644 --- a/tests/test_pagure_flask_dump_load_ticket.py +++ b/tests/test_pagure_flask_dump_load_ticket.py @@ -56,8 +56,10 @@ class PagureFlaskDumpLoadTicketTests(tests.Modeltests): self.app = pagure.APP.test_client() @patch('pagure.lib.notify.send_email') - def test_dumping_reloading_ticket(self, send_email): + @patch('pagure.lib.git._maybe_wait') + def test_dumping_reloading_ticket(self, mw, send_email): """ Test dumping a ticket into a JSON blob. """ + mw.side_effect = lambda result: result.get() send_email.return_value = True tests.create_projects(self.session) @@ -199,6 +201,12 @@ class PagureFlaskDumpLoadTicketTests(tests.Modeltests): self.setUp() tests.create_projects(self.session) + # Create repo + self.gitrepo = os.path.join(self.path, 'tickets', 'test.git') + repopath = os.path.join(self.path, 'tickets') + os.makedirs(self.gitrepo) + pygit2.init_repository(self.gitrepo, bare=True) + pagure.lib.git.update_ticket_from_git( self.session, reponame='test', From e56b253c86841548e89a578287391500e0447555 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 45/57] Small fixes Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/api/__init__.py b/pagure/api/__init__.py index def1205..a07f195 100644 --- a/pagure/api/__init__.py +++ b/pagure/api/__init__.py @@ -28,7 +28,7 @@ API = flask.Blueprint('api_ns', __name__, url_prefix='/api/0') import pagure # noqa: E402 import pagure.lib # noqa: E402 -import pagure.lib.tasks +import pagure.lib.tasks # noqa: E402 from pagure import __api_version__, APP, SESSION, authenticated # noqa: E402 from pagure.doc_utils import load_doc, modify_rst, modify_html # noqa: E402 from pagure.exceptions import APIError # noqa: E402 @@ -324,7 +324,7 @@ def api_task_status(taskid): output = {'ready': False, 'status': result.status} else: - output = {'ready': True, + output = {'ready': True, 'succesful': result.succesful(), 'status': result.status} diff --git a/pagure/ui/app.py b/pagure/ui/app.py index a771613..5b9bc41 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -492,7 +492,6 @@ def wait_task(taskid): result = pagure.lib.tasks.get_result(taskid) if result.ready: result = result.get(timeout=0) - print result endpoint = result.pop('endpoint') return flask.redirect( flask.url_for(endpoint, **result)) diff --git a/tests/test_pagure_flask_ui_app.py b/tests/test_pagure_flask_ui_app.py index d7863e7..d76444e 100644 --- a/tests/test_pagure_flask_ui_app.py +++ b/tests/test_pagure_flask_ui_app.py @@ -274,7 +274,7 @@ class PagureFlaskApptests(tests.Modeltests): user.username = 'foo' with tests.user_set(pagure.APP, user): - data['csrf_token'] = csrf_token + data['csrf_token'] = csrf_token output = self.app.post('/new/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) self.assertIn( From 14c5406562569428e4df4b12f2e67437ab1a199f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 46/57] Improve error reporting for tasks Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/__init__.py b/pagure/__init__.py index 4f134c3..d4d0fe7 100644 --- a/pagure/__init__.py +++ b/pagure/__init__.py @@ -673,9 +673,13 @@ def get_remote_repo_path(remote_git, branch_from, ignore_non_exist=False): return repopath -def wait_for_task(taskid): +def wait_for_task(taskid, prev=None): + if prev is None: + prev = flask.request.full_path return flask.redirect(flask.url_for( - 'wait_task', taskid=taskid)) + 'wait_task', + taskid=taskid, + prev=prev)) def wait_for_task_post(taskid, form, endpoint, initial=False, **kwargs): diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 5b9bc41..2a66165 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -489,10 +489,18 @@ def new_project(): @APP.route('/wait/') def wait_task(taskid): - result = pagure.lib.tasks.get_result(taskid) - if result.ready: - result = result.get(timeout=0) + status = pagure.lib.tasks.get_result(taskid) + if status.ready: + result = status.get(timeout=0, propagate=False) + if status.failed(): + flask.flash('Your task failed: %s' % str(result)) + status.forget() + prev = flask.request.args.get('prev') + if not prev or not prev.startswith('/'): + prev = '/' + return flask.redirect(prev) endpoint = result.pop('endpoint') + status.forget() return flask.redirect( flask.url_for(endpoint, **result)) else: diff --git a/pagure/ui/fork.py b/pagure/ui/fork.py index def7cb2..78be8ef 100644 --- a/pagure/ui/fork.py +++ b/pagure/ui/fork.py @@ -727,7 +727,13 @@ def merge_request_pull(repo, requestid, username=None, namespace=None): taskid = pagure.lib.tasks.merge_pull_request.delay( repo.name, namespace, username, requestid, flask.g.fas_user.username) - return pagure.wait_for_task(taskid) + return pagure.wait_for_task( + taskid, + prev=flask.url_for('request_pull', + repo=repo.name, + namespace=namespace, + username=username, + requestid=requestid)) except pygit2.GitError as err: _log.info('GitError exception raised') flask.flash(str(err.message), 'error') From 3a11db03ebb0850200c45522ea0dfbeeb41ee96f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 47/57] Enable error emailing from the workers Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/mail_logging.py b/pagure/mail_logging.py index 32bbb9d..c1b7f95 100644 --- a/pagure/mail_logging.py +++ b/pagure/mail_logging.py @@ -88,20 +88,27 @@ class ContextInjector(logging.Filter): # pragma: no cover record.callstack = self.format_callstack() - record.url = getattr(flask.request, 'url', '-') - record.args = getattr(flask.request, 'args', '-') - record.form = '-' - record.username = '-' try: - record.form = dict(flask.request.form) - if 'csrf_token' in record.form: - record.form['csrf_token'] = 'Was present, is cleaned up' + record.url = getattr(flask.request, 'url', '-') + record.args = getattr(flask.request, 'args', '-') + record.form = '-' + record.username = '-' + try: + record.form = dict(flask.request.form) + if 'csrf_token' in record.form: + record.form['csrf_token'] = 'Was present, is cleaned up' + except RuntimeError: + pass + try: + record.username = flask.g.fas_user.username + except: + pass except RuntimeError: - pass - try: - record.username = flask.g.fas_user.username - except: - pass + # This means we are sending an error email from the worker + record.url = '* Worker *' + record.args = '' + record.form = '-' + record.username = '-' return True From bb2453be8186cf4a34362110339bcd685e6e602f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 48/57] Update expected error message Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/test_pagure_flask_ui_fork.py b/tests/test_pagure_flask_ui_fork.py index c1dff6e..54088ec 100644 --- a/tests/test_pagure_flask_ui_fork.py +++ b/tests/test_pagure_flask_ui_fork.py @@ -493,9 +493,7 @@ class PagureFlaskForktests(tests.Modeltests): '

PR#1\n' ' PR from the feature branch\n ', output.data) - self.assertIn( - '\n Merge conflicts!', - output.data) + self.assertIn('Merge conflicts!', output.data) @patch('pagure.lib.notify.send_email') def test_merge_request_pull_nochange(self, send_email): @@ -529,9 +527,8 @@ class PagureFlaskForktests(tests.Modeltests): '

PR#1\n' ' Merged', output.data) - self.assertIn( - '\n Nothing to do, changes ' - 'were already merged', output.data) + self.assertIn('Nothing to do, changes were already merged', + output.data) # Check if the closing notification was added output = self.app.get('/test/pull-request/1') From fce947ff08a7e4b9c4d7d0935ba0aa6c17b47b9e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 49/57] Fix clean_git Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 2a4ce2f..989323b 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -323,14 +323,12 @@ def _update_git(obj, repo, repofolder): def clean_git(obj, repo, repofolder): - user = None ticketuid = obj.uid - if repo.is_fork: - user = repo.user - return pagure.lib.tasks.clean_git.delay( - repo.name, repo.namespace, user.username, ticketuid) + repo.name, repo.namespace, + repo.user.username if repo.is_fork else None, + ticketuid) def _clean_git(obj, repo, repofolder): From b235dcdd0402abca53b61098f80c7f2d10ad77fc Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 50/57] Fix add_attachment Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py index 0aca725..b7275c5 100644 --- a/pagure/lib/__init__.py +++ b/pagure/lib/__init__.py @@ -2521,7 +2521,7 @@ def add_attachment(repo, issue, attachmentfolder, user, filename, filestream): tasks.add_file_to_git.delay( repo.name, repo.namespace, repo.user.username if repo.is_fork else None, - user, issue.uid, filename) + user.username, issue.uid, filename) return filename diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 26dd747..1d9dfb3 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -347,6 +347,7 @@ def add_file_to_git(name, namespace, user, user_attacher, issueuid, filename): project = pagure.lib._get_project(session, namespace=namespace, name=name, user=user) issue = pagure.lib.get_issue_by_uid(session, issueuid) + user_attacher = pagure.lib.search_user(session, username=user_attacher) pagure.lib.git._add_file_to_git( project, issue, APP.config['ATTACHMENTS_FOLDER'], From 84b223aadb85b41dca44611b15421e35d1c5a6be Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 51/57] Ignore empty repofolder Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/lib/git.py b/pagure/lib/git.py index 989323b..6d79166 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -323,6 +323,9 @@ def _update_git(obj, repo, repofolder): def clean_git(obj, repo, repofolder): + if not repofolder: + return + ticketuid = obj.uid return pagure.lib.tasks.clean_git.delay( From 1709741885abe74c0d5083edfcad4da5c59cb0e9 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 52/57] Make all tests use self.path/repos for repos Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index fa181e8..841e503 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -151,7 +151,7 @@ class Modeltests(unittest.TestCase): self.path = tempfile.mkdtemp(prefix='pagure-tests-path-') LOG.info('Testdir: %s', self.path) for folder in ['tickets', 'repos', 'forks', 'docs', 'requests', - 'releases', 'remotes']: + 'releases', 'remotes', 'attachments']: os.mkdir(os.path.join(self.path, folder)) if DB_PATH: @@ -235,13 +235,16 @@ class Modeltests(unittest.TestCase): # Prevent unit-tests to send email, globally pagure.APP.config['EMAIL_SEND'] = False pagure.APP.config['TESTING'] = True - pagure.APP.config['GIT_FOLDER'] = self.path + pagure.APP.config['GIT_FOLDER'] = os.path.join( + self.path, 'repos') pagure.APP.config['TICKETS_FOLDER'] = os.path.join( self.path, 'tickets') pagure.APP.config['DOCS_FOLDER'] = os.path.join( self.path, 'docs') pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( self.path, 'requests') + pagure.APP.config['ATTACHMENTS_FOLDER'] = os.path.join( + self.path, 'attachments') self.app = pagure.APP.test_client() def tearDown(self): # pylint: disable=invalid-name diff --git a/tests/test_pagure_exclude_group_index.py b/tests/test_pagure_exclude_group_index.py index 37cf7d3..9a02c49 100644 --- a/tests/test_pagure_exclude_group_index.py +++ b/tests/test_pagure_exclude_group_index.py @@ -33,7 +33,6 @@ class PagureExcludeGroupIndex(tests.Modeltests): """ Set up the environnment, ran before every tests. """ super(PagureExcludeGroupIndex, self).setUp() - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') tests.create_projects(self.session) tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) diff --git a/tests/test_pagure_flask.py b/tests/test_pagure_flask.py index b2358c5..4d789de 100644 --- a/tests/test_pagure_flask.py +++ b/tests/test_pagure_flask.py @@ -34,9 +34,6 @@ class PagureGetRemoteRepoPath(tests.Modeltests): """ Set up the environnment, ran before every tests. """ super(PagureGetRemoteRepoPath, self).setUp() - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') - pagure.APP.config['REMOTE_GIT_FOLDER'] = os.path.join( - self.path, 'remotes') tests.create_projects(self.session) tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) tests.add_content_git_repo(os.path.join(self.path, 'repos', 'test2.git')) diff --git a/tests/test_pagure_flask_api_project.py b/tests/test_pagure_flask_api_project.py index d7b4883..990bea9 100644 --- a/tests/test_pagure_flask_api_project.py +++ b/tests/test_pagure_flask_api_project.py @@ -44,14 +44,6 @@ class PagureFlaskApiProjecttests(tests.Modeltests): pagure.api.project.SESSION = self.session pagure.lib.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - self.app = pagure.APP.test_client() def test_api_git_tags(self): diff --git a/tests/test_pagure_flask_api_ui_private_repo.py b/tests/test_pagure_flask_api_ui_private_repo.py index 5f60edd..29f90c0 100644 --- a/tests/test_pagure_flask_api_ui_private_repo.py +++ b/tests/test_pagure_flask_api_ui_private_repo.py @@ -233,15 +233,6 @@ class PagurePrivateRepotest(tests.Modeltests): pagure.api.issue.SESSION = self.session pagure.APP.config['VIRUS_SCAN_ATTACHMENTS'] = False - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['UPLOAD_FOLDER_PATH'] = os.path.join( - self.path, 'releases') self.app = pagure.APP.test_client() def set_up_git_repo( @@ -251,7 +242,7 @@ class PagurePrivateRepotest(tests.Modeltests): """ # Create a git repo to play with - gitrepo = os.path.join(self.path, 'pmc.git') + gitrepo = os.path.join(self.path, 'repos', 'pmc.git') repo = pygit2.init_repository(gitrepo, bare=True) newpath = tempfile.mkdtemp(prefix='pagure-private-test') @@ -908,11 +899,11 @@ class PagurePrivateRepotest(tests.Modeltests): self.session.commit() # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test4.git') + gitrepo = os.path.join(self.path, 'repos', 'test4.git') repo = pygit2.init_repository(gitrepo, bare=True) newpath = tempfile.mkdtemp(prefix='pagure-fork-test') - repopath = os.path.join(newpath, 'test4') + repopath = os.path.join(newpath, 'repos', 'test4') clone_repo = pygit2.clone_repository(gitrepo, repopath) # Create a file in that git repo with open(os.path.join(repopath, 'sources'), 'w') as stream: diff --git a/tests/test_pagure_flask_docs.py b/tests/test_pagure_flask_docs.py index 37bd6c3..90db374 100644 --- a/tests/test_pagure_flask_docs.py +++ b/tests/test_pagure_flask_docs.py @@ -47,17 +47,13 @@ class PagureFlaskDocstests(tests.Modeltests): pagure.ui.app.SESSION = self.session pagure.ui.repo.SESSION = self.session - pagure.docs_server.APP.config['GIT_FOLDER'] = self.path + pagure.docs_server.APP.config['GIT_FOLDER'] = os.path.join( + self.path, 'repos') pagure.docs_server.APP.config['TICKETS_FOLDER'] = os.path.join( self.path, 'tickets') pagure.docs_server.APP.config['DOCS_FOLDER'] = os.path.join( self.path, 'docs') - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.docs_server.APP.test_client() def _set_up_doc(self): diff --git a/tests/test_pagure_flask_dump_load_ticket.py b/tests/test_pagure_flask_dump_load_ticket.py index 83809b9..7391336 100644 --- a/tests/test_pagure_flask_dump_load_ticket.py +++ b/tests/test_pagure_flask_dump_load_ticket.py @@ -46,13 +46,6 @@ class PagureFlaskDumpLoadTicketTests(tests.Modeltests): pagure.ui.fork.SESSION = self.session pagure.ui.repo.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') self.app = pagure.APP.test_client() @patch('pagure.lib.notify.send_email') diff --git a/tests/test_pagure_flask_internal.py b/tests/test_pagure_flask_internal.py index b83cf2e..487cc2c 100644 --- a/tests/test_pagure_flask_internal.py +++ b/tests/test_pagure_flask_internal.py @@ -43,7 +43,8 @@ class PagureFlaskInternaltests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path + pagure.APP.config['GIT_FOLDER'] = os.path.join( + self.path, 'repos') pagure.APP.config['REQUESTS_FOLDER'] = None pagure.APP.config['TICKETS_FOLDER'] = None pagure.APP.config['DOCS_FOLDER'] = None @@ -437,7 +438,7 @@ class PagureFlaskInternaltests(tests.Modeltests): # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') self.assertFalse(os.path.exists(gitrepo)) os.makedirs(gitrepo) repo = pygit2.init_repository(gitrepo) @@ -559,7 +560,7 @@ class PagureFlaskInternaltests(tests.Modeltests): # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') self.assertFalse(os.path.exists(gitrepo)) os.makedirs(gitrepo) repo = pygit2.init_repository(gitrepo) @@ -681,7 +682,7 @@ class PagureFlaskInternaltests(tests.Modeltests): # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') self.assertFalse(os.path.exists(gitrepo)) os.makedirs(gitrepo) repo = pygit2.init_repository(gitrepo) @@ -822,7 +823,7 @@ class PagureFlaskInternaltests(tests.Modeltests): # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') self.assertFalse(os.path.exists(gitrepo)) os.makedirs(gitrepo) repo = pygit2.init_repository(gitrepo) @@ -957,7 +958,7 @@ class PagureFlaskInternaltests(tests.Modeltests): def test_get_branches_of_commit(self): ''' Test the get_branches_of_commit from the internal API. ''' tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser() user.username = 'pingou' @@ -1038,7 +1039,7 @@ class PagureFlaskInternaltests(tests.Modeltests): ) # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') self.assertTrue(os.path.exists(gitrepo)) repo = pygit2.Repository(gitrepo) diff --git a/tests/test_pagure_flask_ui_app.py b/tests/test_pagure_flask_ui_app.py index d76444e..e5de177 100644 --- a/tests/test_pagure_flask_ui_app.py +++ b/tests/test_pagure_flask_ui_app.py @@ -43,13 +43,6 @@ class PagureFlaskApptests(tests.Modeltests): pagure.ui.filters.SESSION = self.session pagure.ui.repo.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') self.app = pagure.APP.test_client() def test_index(self): @@ -173,7 +166,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) self.assertFalse(os.path.exists( - os.path.join(self.path, 'project-1.git'))) + os.path.join(self.path, 'repos', 'project-1.git'))) self.assertFalse(os.path.exists( os.path.join(self.path, 'tickets', 'project-1.git'))) self.assertFalse(os.path.exists( @@ -209,7 +202,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) self.assertFalse(os.path.exists( - os.path.join(self.path, 'project-1.git'))) + os.path.join(self.path, 'repos', 'project-1.git'))) self.assertFalse(os.path.exists( os.path.join(self.path, 'tickets', 'project-1.git'))) self.assertFalse(os.path.exists( @@ -225,7 +218,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) self.assertFalse(os.path.exists( - os.path.join(self.path, 'project#1.git'))) + os.path.join(self.path, 'repos', 'project#1.git'))) self.assertFalse(os.path.exists( os.path.join(self.path, 'tickets', 'project#1.git'))) self.assertFalse(os.path.exists( @@ -288,7 +281,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 1) self.assertTrue(os.path.exists( - os.path.join(self.path, 'project-1.git'))) + os.path.join(self.path, 'repos', 'project-1.git'))) self.assertTrue(os.path.exists( os.path.join(self.path, 'tickets', 'project-1.git'))) self.assertTrue(os.path.exists( @@ -303,7 +296,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) self.assertFalse(os.path.exists( - os.path.join(self.path, 'foo', 'project#1.git'))) + os.path.join(self.path, 'repos', 'foo', 'project#1.git'))) self.assertFalse(os.path.exists( os.path.join(self.path, 'tickets', 'foo', 'project#1.git'))) self.assertFalse(os.path.exists( @@ -369,7 +362,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session, private=True) self.assertEqual(len(projects), 1) self.assertTrue(os.path.exists( - os.path.join(self.path, 'foo', 'project-1.git'))) + os.path.join(self.path, 'repos', 'foo', 'project-1.git'))) self.assertTrue(os.path.exists( os.path.join(self.path, 'tickets', 'foo', 'project-1.git'))) self.assertTrue(os.path.exists( @@ -383,7 +376,7 @@ class PagureFlaskApptests(tests.Modeltests): projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) self.assertFalse(os.path.exists( - os.path.join(self.path, 'project-1.git'))) + os.path.join(self.path, 'repos', 'project-1.git'))) self.assertFalse(os.path.exists( os.path.join(self.path, 'tickets', 'project-1.git'))) self.assertFalse(os.path.exists( @@ -441,7 +434,7 @@ class PagureFlaskApptests(tests.Modeltests): self.assertEqual(len(projects), 2) for project in ['project-1', 'project-2']: self.assertTrue(os.path.exists( - os.path.join(self.path, '%s.git' % project))) + os.path.join(self.path, 'repos', '%s.git' % project))) self.assertTrue(os.path.exists( os.path.join(self.path, 'tickets', '%s.git' % project))) self.assertTrue(os.path.exists( @@ -628,7 +621,7 @@ class PagureFlaskApptests(tests.Modeltests): 'name="csrf_token" type="hidden" value="')[1].split('">')[0] tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) text = 'Cf commit 9364354a4555ba17aa60f0d' exp = '

Cf commit 9364354a4555ba17aa60f0d

' @@ -657,8 +650,8 @@ class PagureFlaskApptests(tests.Modeltests): 'name="csrf_token" type="hidden" value="')[1].split('">')[0] tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) - repopath = os.path.join(self.path, 'test.git') + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) + repopath = os.path.join(self.path, 'repos', 'test.git') tests.add_content_git_repo(repopath) repo = pygit2.Repository(repopath) diff --git a/tests/test_pagure_flask_ui_app_give_project.py b/tests/test_pagure_flask_ui_app_give_project.py index b7554f3..5121bb9 100644 --- a/tests/test_pagure_flask_ui_app_give_project.py +++ b/tests/test_pagure_flask_ui_app_give_project.py @@ -42,20 +42,13 @@ class PagureFlaskGiveRepotests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.APP.config['VIRUS_SCAN_ATTACHMENTS'] = False - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') pagure.APP.config['UPLOAD_FOLDER_URL'] = '/releases/' pagure.APP.config['UPLOAD_FOLDER_PATH'] = os.path.join( self.path, 'releases') self.app = pagure.APP.test_client() tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) def _check_user(self, user='pingou'): project = pagure.get_authorized_project( diff --git a/tests/test_pagure_flask_ui_fork.py b/tests/test_pagure_flask_ui_fork.py index 54088ec..771bf48 100644 --- a/tests/test_pagure_flask_ui_fork.py +++ b/tests/test_pagure_flask_ui_fork.py @@ -66,13 +66,6 @@ class PagureFlaskForktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.issues.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') self.app = pagure.APP.test_client() def set_up_git_repo( @@ -399,9 +392,6 @@ class PagureFlaskForktests(tests.Modeltests): self.assertIn( 'Overview - test - Pagure', output.data) self.assertIn( - '\n Changes merged!', - output.data) - self.assertIn( 'A commit on branch feature', output.data) self.assertNotIn( 'Merge #1 `PR from the feature branch`', output.data) @@ -451,9 +441,6 @@ class PagureFlaskForktests(tests.Modeltests): self.assertEqual(output.status_code, 200) self.assertIn( 'Overview - test - Pagure', output.data) - self.assertIn( - '\n Changes merged!', - output.data) # Check if the closing notification was added output = self.app.get('/test/pull-request/1') @@ -1354,9 +1341,6 @@ index 0000000..2a552bb output = self.app.post( '/do_fork/test', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) - self.assertIn( - '\n Repo "test" ' - 'cloned to "foo/test"', output.data) @patch('pagure.lib.notify.send_email') def test_new_request_pull(self, send_email): @@ -1891,9 +1875,6 @@ index 0000000..2a552bb self.assertIn( 'Overview - test - Pagure', output.data) self.assertIn( - '\n Changes merged!', - output.data) - self.assertIn( 'Merge #1 `PR from the feature branch`', output.data) self.assertIn( 'A commit on branch feature', output.data) @@ -2134,9 +2115,6 @@ index 0000000..2a552bb self.assertIn( '', output.data) - self.assertIn( - '\n Repo "test" ' - 'cloned to "foo/test"', output.data) # View what's supposed to be an image output = self.app.post('fork_edit/test/edit/master/f/test.jpg', diff --git a/tests/test_pagure_flask_ui_groups.py b/tests/test_pagure_flask_ui_groups.py index 2d9fb76..2f5cb92 100644 --- a/tests/test_pagure_flask_ui_groups.py +++ b/tests/test_pagure_flask_ui_groups.py @@ -41,13 +41,6 @@ class PagureFlaskGroupstests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') self.app = pagure.APP.test_client() def test_group_lists(self): diff --git a/tests/test_pagure_flask_ui_issues.py b/tests/test_pagure_flask_ui_issues.py index 5131e1f..c676904 100644 --- a/tests/test_pagure_flask_ui_issues.py +++ b/tests/test_pagure_flask_ui_issues.py @@ -52,11 +52,6 @@ class PagureFlaskIssuestests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() @patch('pagure.lib.git.update_git') @@ -77,7 +72,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/new_issue') self.assertEqual(output.status_code, 200) @@ -166,7 +161,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) tests.create_projects_git( os.path.join(self.path, 'tickets'), bare=True) @@ -206,7 +201,7 @@ class PagureFlaskIssuestests(tests.Modeltests): output.data) # Check the image was uploaded self.assertIn( - 'href="/test/issue/raw/files/' + 'href="/test/issue/raw/' '8a06845923010b27bfd8e7e75acff7badc40d1021b4' '994e01f5e11ca40bc3abe', output.data) @@ -271,7 +266,7 @@ class PagureFlaskIssuestests(tests.Modeltests): output.data) # Check the image was uploaded self.assertIn( - 'href="/somenamespace/test3/issue/raw/files/' + 'href="/somenamespace/test3/issue/raw/' '8a06845923010b27bfd8e7e75acff7badc40d1021b4' '994e01f5e11ca40bc3abe', output.data) @@ -288,7 +283,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issues') self.assertEqual(output.status_code, 200) @@ -491,7 +486,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -621,7 +616,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -700,7 +695,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -820,7 +815,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -887,7 +882,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) repo = pagure.get_authorized_project(self.session, 'test') @@ -947,7 +942,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1/update') self.assertEqual(output.status_code, 302) @@ -1257,7 +1252,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) # Create issues to play with repo = pagure.get_authorized_project(self.session, 'test') @@ -1367,7 +1362,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) # Create issues to play with repo = pagure.get_authorized_project(self.session, 'test') @@ -1460,7 +1455,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) # Create issues to play with repo = pagure.get_authorized_project(self.session, 'test') @@ -1584,7 +1579,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) tests.create_projects_git( os.path.join(self.path, 'tickets'), bare=True) @@ -1671,7 +1666,7 @@ class PagureFlaskIssuestests(tests.Modeltests): os.path.abspath(__file__))[1:].replace('/', '_') exp = { 'output': 'ok', - 'filelocation': '/test/issue/raw/files/8a06845923010b27bfd8' + 'filelocation': '/test/issue/raw/8a06845923010b27bfd8' 'e7e75acff7badc40d1021b4994e01f5e11ca40bc3a' 'be-%s_placebo.png' % folder, 'filename': '%s_placebo.png' % folder, @@ -1708,7 +1703,7 @@ class PagureFlaskIssuestests(tests.Modeltests): self.session.commit() self.assertEqual(msg.title, 'Test issue') - url = '/issue/raw/files/8a06845923010b27bfd8'\ + url = '/issue/raw/8a06845923010b27bfd8'\ 'e7e75acff7badc40d1021b4994e01f5e11ca40bc3a'\ 'be-home_pierrey_repos_gitrepo_pagure_tests'\ '_placebo.png' @@ -1739,7 +1734,7 @@ class PagureFlaskIssuestests(tests.Modeltests): self.session.add(repo) self.session.commit() - url = '/issue/raw/files/8a06845923010b27bfd8'\ + url = '/issue/raw/8a06845923010b27bfd8'\ 'e7e75acff7badc40d1021b4994e01f5e11ca40bc3a'\ 'be-%s_placebo.png' % os.path.dirname( os.path.abspath(__file__))[1:].replace('/', '_') @@ -1747,7 +1742,7 @@ class PagureFlaskIssuestests(tests.Modeltests): output = self.app.get('/foo' + url) self.assertEqual(output.status_code, 404) - output = self.app.get('/test/issue/raw/files/test.png') + output = self.app.get('/test/issue/raw/test.png') self.assertEqual(output.status_code, 404) # Access file by name @@ -1781,7 +1776,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1/edit') self.assertEqual(output.status_code, 404) @@ -1896,7 +1891,7 @@ class PagureFlaskIssuestests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.get('/test/tag/foo/edit') self.assertEqual(output.status_code, 403) @@ -1999,7 +1994,7 @@ class PagureFlaskIssuestests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/droptag/') self.assertEqual(output.status_code, 403) @@ -2073,7 +2068,7 @@ class PagureFlaskIssuestests(tests.Modeltests): p_ugt.return_value = True tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git(os.path.join(self.path, 'tickets')) # Create issues to play with @@ -2156,7 +2151,7 @@ class PagureFlaskIssuestests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) # Create issues to play with repo = pagure.get_authorized_project(self.session, 'test') @@ -2438,7 +2433,7 @@ class PagureFlaskIssuestests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/update/tags') self.assertEqual(output.status_code, 403) diff --git a/tests/test_pagure_flask_ui_issues_acl_checks.py b/tests/test_pagure_flask_ui_issues_acl_checks.py index 7732164..88bf61d 100644 --- a/tests/test_pagure_flask_ui_issues_acl_checks.py +++ b/tests/test_pagure_flask_ui_issues_acl_checks.py @@ -49,11 +49,6 @@ class PagureFlaskIssuesACLtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() @patch('pagure.lib.git.update_git') @@ -68,7 +63,7 @@ class PagureFlaskIssuesACLtests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -319,7 +314,7 @@ class PagureFlaskIssuesACLtests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -582,7 +577,7 @@ class PagureFlaskIssuesACLtests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) @@ -844,7 +839,7 @@ class PagureFlaskIssuesACLtests(tests.Modeltests): tests.create_projects(self.session) tests.create_projects_git( - os.path.join(self.path), bare=True) + os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/issue/1') self.assertEqual(output.status_code, 404) diff --git a/tests/test_pagure_flask_ui_no_master_branch.py b/tests/test_pagure_flask_ui_no_master_branch.py index 5a7a66b..733b33d 100644 --- a/tests/test_pagure_flask_ui_no_master_branch.py +++ b/tests/test_pagure_flask_ui_no_master_branch.py @@ -45,13 +45,6 @@ class PagureFlaskNoMasterBranchtests(tests.Modeltests): pagure.ui.fork.SESSION = self.session pagure.ui.repo.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = os.path.join(self.path, 'repos') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') self.app = pagure.APP.test_client() def set_up_git_repo(self): diff --git a/tests/test_pagure_flask_ui_old_commit.py b/tests/test_pagure_flask_ui_old_commit.py index d8bce31..c97b75a 100644 --- a/tests/test_pagure_flask_ui_old_commit.py +++ b/tests/test_pagure_flask_ui_old_commit.py @@ -46,13 +46,6 @@ class PagureFlaskRepoOldUrltests(tests.Modeltests): pagure.APP.config['OLD_VIEW_COMMIT_ENABLED'] = True pagure.APP.config['EMAIL_SEND'] = False - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') pagure.APP.config['UPLOAD_FOLDER_PATH'] = os.path.join( self.path, 'releases') self.app = pagure.APP.test_client() @@ -68,11 +61,11 @@ class PagureFlaskRepoOldUrltests(tests.Modeltests): """ Test the view_commit_old endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Add a README to the git repo - First commit - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View first commit @@ -118,9 +111,10 @@ class PagureFlaskRepoOldUrltests(tests.Modeltests): '#ddffdd">+ ======' in output.data) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View another commit @@ -158,7 +152,7 @@ class PagureFlaskRepoOldUrltests(tests.Modeltests): self.session.add(item) self.session.commit() forkedgit = os.path.join( - self.path, 'forks', 'pingou', 'test3.git') + self.path, 'repos', 'forks', 'pingou', 'test3.git') tests.add_content_git_repo(forkedgit) tests.add_readme_git_repo(forkedgit) diff --git a/tests/test_pagure_flask_ui_plugins.py b/tests/test_pagure_flask_ui_plugins.py index d572f1c..aed76e0 100644 --- a/tests/test_pagure_flask_ui_plugins.py +++ b/tests/test_pagure_flask_ui_plugins.py @@ -55,11 +55,6 @@ class PagureFlaskPluginstests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_get_plugin_names(self): @@ -92,7 +87,7 @@ class PagureFlaskPluginstests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.get('/test/settings/Mail') self.assertEqual(output.status_code, 403) diff --git a/tests/test_pagure_flask_ui_plugins_default_hook.py b/tests/test_pagure_flask_ui_plugins_default_hook.py index 4fae359..fee7f9b 100644 --- a/tests/test_pagure_flask_ui_plugins_default_hook.py +++ b/tests/test_pagure_flask_ui_plugins_default_hook.py @@ -42,20 +42,13 @@ class PagureFlaskPluginDefaultHooktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_default_ui(self): """ Test the default hook plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -67,13 +60,13 @@ class PagureFlaskPluginDefaultHooktests(tests.Modeltests): project is created. """ - msg = pagure.lib.new_project( + taskid = pagure.lib.new_project( self.session, user='pingou', name='test', blacklist=[], allowed_prefix=[], - gitfolder=self.path, + gitfolder=os.path.join(self.path, 'repos'), docfolder=os.path.join(self.path, 'docs'), ticketfolder=os.path.join(self.path, 'tickets'), requestfolder=os.path.join(self.path, 'requests'), @@ -85,13 +78,16 @@ class PagureFlaskPluginDefaultHooktests(tests.Modeltests): prevent_40_chars=False, namespace=None ) - - self.assertEqual(msg, 'Project "test" created') + result = pagure.lib.tasks.get_result(taskid).get() + self.assertEqual(result, + {'endpoint': 'view_repo', + 'repo': 'test', + 'namespace': None}) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.default'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.default'))) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive'))) if __name__ == '__main__': diff --git a/tests/test_pagure_flask_ui_plugins_fedmsg.py b/tests/test_pagure_flask_ui_plugins_fedmsg.py index db7a8f6..86d7d4a 100644 --- a/tests/test_pagure_flask_ui_plugins_fedmsg.py +++ b/tests/test_pagure_flask_ui_plugins_fedmsg.py @@ -42,18 +42,13 @@ class PagureFlaskPluginFedmsgtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_fedmsg(self): """ Test the fedmsg plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -83,7 +78,8 @@ class PagureFlaskPluginFedmsgtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.fedmsg'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.fedmsg'))) data['csrf_token'] = csrf_token @@ -107,7 +103,8 @@ class PagureFlaskPluginFedmsgtests(tests.Modeltests): output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.fedmsg'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.fedmsg'))) # Activate hook data = { @@ -135,7 +132,8 @@ class PagureFlaskPluginFedmsgtests(tests.Modeltests): 'value="y">' in output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.fedmsg'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.fedmsg'))) # De-Activate hook data = {'csrf_token': csrf_token} @@ -159,7 +157,8 @@ class PagureFlaskPluginFedmsgtests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.fedmsg'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.fedmsg'))) if __name__ == '__main__': diff --git a/tests/test_pagure_flask_ui_plugins_irc.py b/tests/test_pagure_flask_ui_plugins_irc.py index 862bc3e..2055a98 100644 --- a/tests/test_pagure_flask_ui_plugins_irc.py +++ b/tests/test_pagure_flask_ui_plugins_irc.py @@ -42,18 +42,13 @@ class PagureFlaskPluginIRCtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_mail(self): """ Test the irc plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -83,7 +78,7 @@ class PagureFlaskPluginIRCtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.irc'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.irc'))) data['csrf_token'] = csrf_token @@ -108,7 +103,7 @@ class PagureFlaskPluginIRCtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.irc'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.irc'))) # Activate hook data = { @@ -140,7 +135,7 @@ class PagureFlaskPluginIRCtests(tests.Modeltests): # TODO: Fix this #self.assertTrue(os.path.exists(os.path.join( - #self.path, 'test.git', 'hooks', 'post-receive.irc'))) + #self.path, 'repos', 'test.git', 'hooks', 'post-receive.irc'))) # De-Activate hook data = {'csrf_token': csrf_token} @@ -163,7 +158,7 @@ class PagureFlaskPluginIRCtests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.irc'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.irc'))) if __name__ == '__main__': diff --git a/tests/test_pagure_flask_ui_plugins_mail.py b/tests/test_pagure_flask_ui_plugins_mail.py index 0f29008..2607093 100644 --- a/tests/test_pagure_flask_ui_plugins_mail.py +++ b/tests/test_pagure_flask_ui_plugins_mail.py @@ -42,18 +42,13 @@ class PagureFlaskPluginMailtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_mail(self): """ Test the mail plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -114,7 +109,7 @@ class PagureFlaskPluginMailtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.mail'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.mail'))) # Missing the required mail_to data = {'csrf_token': csrf_token, 'active': 'y'} @@ -137,7 +132,7 @@ class PagureFlaskPluginMailtests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.mail'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.mail'))) # Activate hook data = { @@ -168,7 +163,7 @@ class PagureFlaskPluginMailtests(tests.Modeltests): 'value="y">' in output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.mail'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.mail'))) # De-Activate hook data = {'csrf_token': csrf_token} @@ -194,7 +189,7 @@ class PagureFlaskPluginMailtests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.mail'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.mail'))) if __name__ == '__main__': diff --git a/tests/test_pagure_flask_ui_plugins_noff.py b/tests/test_pagure_flask_ui_plugins_noff.py index bdbb40c..e185e07 100644 --- a/tests/test_pagure_flask_ui_plugins_noff.py +++ b/tests/test_pagure_flask_ui_plugins_noff.py @@ -42,18 +42,13 @@ class PagureFlaskPluginNoFFtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_noff(self): """ Test the noff plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -125,7 +120,7 @@ class PagureFlaskPluginNoFFtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.mail'))) + self.path, 'repos', 'test.git', 'hooks', 'post-receive.mail'))) # Missing the required mail_to data = {'csrf_token': csrf_token, 'active': 'y'} @@ -151,7 +146,7 @@ class PagureFlaskPluginNoFFtests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureforcecommit'))) # Activate hook @@ -188,7 +183,7 @@ class PagureFlaskPluginNoFFtests(tests.Modeltests): 'value="y">', output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureforcecommit'))) # De-Activate hook @@ -220,7 +215,7 @@ class PagureFlaskPluginNoFFtests(tests.Modeltests): 'value="y">', output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureforcecommit'))) diff --git a/tests/test_pagure_flask_ui_plugins_pagure_ci.py b/tests/test_pagure_flask_ui_plugins_pagure_ci.py index f35a60e..e92605d 100644 --- a/tests/test_pagure_flask_ui_plugins_pagure_ci.py +++ b/tests/test_pagure_flask_ui_plugins_pagure_ci.py @@ -39,18 +39,13 @@ class PagureFlaskPluginPagureCItests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_pagure_ci(self): """ Test the pagure ci plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -189,7 +184,7 @@ class PagureFlaskPluginPagureCItests(tests.Modeltests): """ Test the pagure ci plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): diff --git a/tests/test_pagure_flask_ui_plugins_pagure_hook.py b/tests/test_pagure_flask_ui_plugins_pagure_hook.py index e702dab..8cd9877 100644 --- a/tests/test_pagure_flask_ui_plugins_pagure_hook.py +++ b/tests/test_pagure_flask_ui_plugins_pagure_hook.py @@ -42,20 +42,13 @@ class PagureFlaskPluginPagureHooktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_mail(self): """ Test the pagure hook plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -111,7 +104,8 @@ class PagureFlaskPluginPagureHooktests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.pagure'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.pagure'))) # Activate hook data = { @@ -140,7 +134,8 @@ class PagureFlaskPluginPagureHooktests(tests.Modeltests): 'value="y">' in output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.pagure'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.pagure'))) # De-Activate hook data = {'csrf_token': csrf_token} @@ -165,7 +160,8 @@ class PagureFlaskPluginPagureHooktests(tests.Modeltests): 'value="y">' in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', 'post-receive.pagure'))) + self.path, 'repos', 'test.git', 'hooks', + 'post-receive.pagure'))) if __name__ == '__main__': diff --git a/tests/test_pagure_flask_ui_plugins_pagure_request_hook.py b/tests/test_pagure_flask_ui_plugins_pagure_request_hook.py index 980dc87..28a639e 100644 --- a/tests/test_pagure_flask_ui_plugins_pagure_request_hook.py +++ b/tests/test_pagure_flask_ui_plugins_pagure_request_hook.py @@ -42,18 +42,13 @@ class PagureFlaskPluginPagureRequestHooktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_pagure_request(self): """ Test the pagure_request plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): diff --git a/tests/test_pagure_flask_ui_plugins_pagure_ticket_hook.py b/tests/test_pagure_flask_ui_plugins_pagure_ticket_hook.py index cb2a1b0..7eb0a75 100644 --- a/tests/test_pagure_flask_ui_plugins_pagure_ticket_hook.py +++ b/tests/test_pagure_flask_ui_plugins_pagure_ticket_hook.py @@ -42,18 +42,13 @@ class PagureFlaskPluginPagureTicketHooktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_pagure_ticket(self): """ Test the pagure_ticket plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): diff --git a/tests/test_pagure_flask_ui_plugins_rtd_hook.py b/tests/test_pagure_flask_ui_plugins_rtd_hook.py index 8408d21..f54fdd4 100644 --- a/tests/test_pagure_flask_ui_plugins_rtd_hook.py +++ b/tests/test_pagure_flask_ui_plugins_rtd_hook.py @@ -42,18 +42,13 @@ class PagureFlaskPluginRtdHooktests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_pagure_request(self): """ Test the pagure_request plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -141,7 +136,7 @@ class PagureFlaskPluginRtdHooktests(tests.Modeltests): 'value="y">', output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'post-receive.rtd'))) # De-Activate hook @@ -168,7 +163,7 @@ class PagureFlaskPluginRtdHooktests(tests.Modeltests): 'value="y">', output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'post-receive.rtd'))) # Try re-activate hook w/o the git repo @@ -177,7 +172,7 @@ class PagureFlaskPluginRtdHooktests(tests.Modeltests): 'active': 'y', 'project_name': 'foo', } - shutil.rmtree(os.path.join(self.path, 'test.git')) + shutil.rmtree(os.path.join(self.path, 'repos', 'test.git')) output = self.app.post('/test/settings/Read the Doc', data=data) self.assertEqual(output.status_code, 404) diff --git a/tests/test_pagure_flask_ui_plugins_unsigned.py b/tests/test_pagure_flask_ui_plugins_unsigned.py index cca5bd3..a4635d9 100644 --- a/tests/test_pagure_flask_ui_plugins_unsigned.py +++ b/tests/test_pagure_flask_ui_plugins_unsigned.py @@ -42,18 +42,13 @@ class PagureFlaskPluginUnsignedtests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.filters.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() def test_plugin_unsigned(self): """ Test the noff plugin on/off endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -116,7 +111,7 @@ class PagureFlaskPluginUnsignedtests(tests.Modeltests): in output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureunsignedcommit'))) # Activate the hook @@ -137,7 +132,7 @@ class PagureFlaskPluginUnsignedtests(tests.Modeltests): output.data) self.assertTrue(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureunsignedcommit'))) # De-Activate hook @@ -166,7 +161,7 @@ class PagureFlaskPluginUnsignedtests(tests.Modeltests): 'value="y">', output.data) self.assertFalse(os.path.exists(os.path.join( - self.path, 'test.git', 'hooks', + self.path, 'repos', 'test.git', 'hooks', 'pre-receive.pagureunsignedcommit'))) diff --git a/tests/test_pagure_flask_ui_priorities.py b/tests/test_pagure_flask_ui_priorities.py index 356b744..278fe4f 100644 --- a/tests/test_pagure_flask_ui_priorities.py +++ b/tests/test_pagure_flask_ui_priorities.py @@ -46,13 +46,6 @@ class PagureFlaskPrioritiestests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.issues.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() @patch('pagure.lib.git.update_git') @@ -63,7 +56,7 @@ class PagureFlaskPrioritiestests(tests.Modeltests): p_ugt.return_value = True tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) user = tests.FakeUser() user.username = 'pingou' @@ -108,7 +101,7 @@ class PagureFlaskPrioritiestests(tests.Modeltests): p_ugt.return_value = True tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Set some priorities repo = pagure.get_authorized_project(self.session, 'test') @@ -154,7 +147,7 @@ class PagureFlaskPrioritiestests(tests.Modeltests): def test_update_priorities(self): """ Test updating priorities of a repo. """ tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Set some priorities repo = pagure.get_authorized_project(self.session, 'test') @@ -361,7 +354,7 @@ class PagureFlaskPrioritiestests(tests.Modeltests): p_ugt.return_value = True tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Start from scrach on priorities repo = pagure.lib._get_project(self.session, 'test') diff --git a/tests/test_pagure_flask_ui_quick_reply.py b/tests/test_pagure_flask_ui_quick_reply.py index 6e9a377..737f2ec 100644 --- a/tests/test_pagure_flask_ui_quick_reply.py +++ b/tests/test_pagure_flask_ui_quick_reply.py @@ -40,14 +40,9 @@ class PagureFlaskQuickReplytest(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.ui.SESSION = self.session - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') self.app = pagure.APP.test_client() tests.create_projects(self.session) - tests.create_projects_git(os.path.join(self.path), bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) self.admin = tests.FakeUser(username='pingou') self.user = tests.FakeUser(username='ralph') diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index 34d14ee..f8581ba 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -46,13 +46,6 @@ class PagureFlaskRepotests(tests.Modeltests): pagure.ui.repo.SESSION = self.session pagure.APP.config['VIRUS_SCAN_ATTACHMENTS'] = False - pagure.APP.config['GIT_FOLDER'] = self.path - pagure.APP.config['REQUESTS_FOLDER'] = os.path.join( - self.path, 'requests') - pagure.APP.config['TICKETS_FOLDER'] = os.path.join( - self.path, 'tickets') - pagure.APP.config['DOCS_FOLDER'] = os.path.join( - self.path, 'docs') pagure.APP.config['UPLOAD_FOLDER_URL'] = '/releases/' pagure.APP.config['UPLOAD_FOLDER_PATH'] = os.path.join( self.path, 'releases') @@ -70,7 +63,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.get('/test/adduser') @@ -102,7 +95,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) data['user'] = 'foo' - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post( '/test/adduser', data=data, follow_redirects=True) self.assertEqual(output.status_code, 404) @@ -120,7 +113,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.get('/test/adddeploykey') @@ -223,7 +216,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.get('/test/adduser') @@ -318,7 +311,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.get('/test/addgroup') @@ -376,7 +369,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.get('/test/addgroup') @@ -482,7 +475,7 @@ class PagureFlaskRepotests(tests.Modeltests): user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/settings') @@ -537,7 +530,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/dropdeploykey/1') self.assertEqual(output.status_code, 403) @@ -613,7 +606,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/dropuser/1') self.assertEqual(output.status_code, 403) @@ -689,7 +682,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # User not logged in output = self.app.post('/test/dropgroup/1') @@ -761,7 +754,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/dropgroup/1') self.assertEqual(output.status_code, 403) @@ -857,7 +850,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # Session timed-out output = self.app.post('/test/update') @@ -950,7 +943,7 @@ class PagureFlaskRepotests(tests.Modeltests): ast.return_value = False tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): @@ -1029,7 +1022,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.get('/test/settings') self.assertEqual(output.status_code, 403) @@ -1161,7 +1154,7 @@ class PagureFlaskRepotests(tests.Modeltests): ) self.session.add(item) self.session.commit() - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.get('/test/settings') self.assertEqual(output.status_code, 403) @@ -1254,7 +1247,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/forks', follow_redirects=True) self.assertEqual(output.status_code, 200) @@ -1275,7 +1268,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.perfMaxWalks(0, 0) self.perfReset() - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test') self.assertEqual(output.status_code, 200) @@ -1296,8 +1289,9 @@ class PagureFlaskRepotests(tests.Modeltests): self.perfReset() # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) self.perfReset() output = self.app.get('/test') @@ -1323,9 +1317,9 @@ class PagureFlaskRepotests(tests.Modeltests): # Add some content to the git repo tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) output = self.app.get('/fork/pingou/test') self.assertEqual(output.status_code, 200) @@ -1350,11 +1344,11 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) output = self.app.get('/fork/pingou/test3') @@ -1372,7 +1366,7 @@ class PagureFlaskRepotests(tests.Modeltests): tests.create_projects(self.session) # Create a git repo to play with - gitrepo = os.path.join(self.path, 'test.git') + gitrepo = os.path.join(self.path, 'repos', 'test.git') pygit2.init_repository(gitrepo, bare=True) # Create a fork of this repo @@ -1428,14 +1422,15 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/branch/master') self.assertEqual(output.status_code, 404) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) output = self.app.get('/test/branch/master') self.assertEqual(output.status_code, 200) @@ -1458,9 +1453,9 @@ class PagureFlaskRepotests(tests.Modeltests): # Add some content to the git repo tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) output = self.app.get('/fork/pingou/test/branch/master') self.assertEqual(output.status_code, 200) @@ -1483,11 +1478,11 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) output = self.app.get('/fork/pingou/test3/branch/master') @@ -1510,7 +1505,7 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/commits') self.assertEqual(output.status_code, 200) @@ -1520,8 +1515,9 @@ class PagureFlaskRepotests(tests.Modeltests): 'test project #1 ', output.data) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) output = self.app.get('/test/commits') self.assertEqual(output.status_code, 200) @@ -1553,9 +1549,9 @@ class PagureFlaskRepotests(tests.Modeltests): # Add some content to the git repo tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test.git')) output = self.app.get('/fork/pingou/test/commits?page=abc') self.assertEqual(output.status_code, 200) @@ -1578,11 +1574,11 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) output = self.app.get('/fork/pingou/test3/commits/fobranch') @@ -1709,26 +1705,26 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/bar') self.assertEqual(output.status_code, 404) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) # Add one commit to git repo tests.add_commit_git_repo( - os.path.join(self.path, 'test.git'), ncommits=1) + os.path.join(self.path, 'repos', 'test.git'), ncommits=1) c1 = repo.revparse_single('HEAD') # Add another commit to git repo tests.add_commit_git_repo( - os.path.join(self.path, 'test.git'), ncommits=1) + os.path.join(self.path, 'repos', 'test.git'), ncommits=1) c2 = repo.revparse_single('HEAD') # Add one more commit to git repo tests.add_commit_git_repo( - os.path.join(self.path, 'test.git'), + os.path.join(self.path, 'repos', 'test.git'), ncommits=1, filename='Šource') c3 = repo.revparse_single('HEAD') @@ -1753,18 +1749,19 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/blob/foo/f/sources') self.assertEqual(output.status_code, 404) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test.jpg') + os.path.join(self.path, 'repos', 'test.git'), 'test.jpg') tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test_binary') + os.path.join(self.path, 'repos', 'test.git'), 'test_binary') output = self.app.get('/test/blob/master/foofile') self.assertEqual(output.status_code, 404) @@ -1781,7 +1778,7 @@ class PagureFlaskRepotests(tests.Modeltests): # Empty files should also be displayed tests.add_content_to_git( - os.path.join(self.path, 'test.git'), + os.path.join(self.path, 'repos', 'test.git'), filename="emptyfile.md", content="") output = self.app.get('/test/blob/master/f/emptyfile.md') @@ -1804,7 +1801,7 @@ class PagureFlaskRepotests(tests.Modeltests): output.data) # View by commit id - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') output = self.app.get('/test/blob/%s/f/test.jpg' % commit.oid.hex) @@ -1855,7 +1852,7 @@ class PagureFlaskRepotests(tests.Modeltests): # View file with a non-ascii name tests.add_commit_git_repo( - os.path.join(self.path, 'test.git'), + os.path.join(self.path, 'repos', 'test.git'), ncommits=1, filename='Šource') output = self.app.get('/test/blob/master/f/Šource') self.assertEqual(output.status_code, 200) @@ -1886,11 +1883,11 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) # Verify the nav links correctly when viewing a file/folder in a fork. @@ -1919,15 +1916,16 @@ class PagureFlaskRepotests(tests.Modeltests): """ Test the view_file endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test.jpg') + os.path.join(self.path, 'repos', 'test.git'), 'test.jpg') tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test_binary') + os.path.join(self.path, 'repos', 'test.git'), 'test_binary') # View file output = self.app.get('/test/blob/master/f/sources') @@ -1946,13 +1944,13 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/raw/foo/sources') self.assertEqual(output.status_code, 404) # Add some content to the git repo - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) # View first commit output = self.app.get('/test/raw/master') @@ -1962,11 +1960,12 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertTrue(':Author: Pierre-Yves Chibon' in output.data) # Add some more content to the repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test.jpg') + os.path.join(self.path, 'repos', 'test.git'), 'test.jpg') tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test_binary') + os.path.join(self.path, 'repos', 'test.git'), 'test_binary') output = self.app.get('/test/raw/master/f/foofile') self.assertEqual(output.status_code, 404) @@ -1984,7 +1983,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertTrue(output.data.startswith('\x00\x00\x01\x00')) # View by commit id - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') output = self.app.get('/test/raw/%s/f/test.jpg' % commit.oid.hex) @@ -2041,11 +2040,11 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) output = self.app.get('/fork/pingou/test3/raw/master/f/sources') @@ -2066,18 +2065,19 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/blame/sources') self.assertEqual(output.status_code, 404) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test.jpg') + os.path.join(self.path, 'repos', 'test.git'), 'test.jpg') tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test_binary') + os.path.join(self.path, 'repos', 'test.git'), 'test_binary') output = self.app.get('/test/blame/foofile') self.assertEqual(output.status_code, 404) @@ -2115,7 +2115,7 @@ class PagureFlaskRepotests(tests.Modeltests): # View file with a non-ascii name tests.add_commit_git_repo( - os.path.join(self.path, 'test.git'), + os.path.join(self.path, 'repos', 'test.git'), ncommits=1, filename='Šource') output = self.app.get('/test/blame/Šource') self.assertEqual(output.status_code, 200) @@ -2146,14 +2146,14 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) tests.add_content_to_git( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), content=u'✨☃🍰☃✨'.encode('utf-8')) output = self.app.get('/fork/pingou/test3/blame/sources') @@ -2177,14 +2177,14 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/c/bar') self.assertEqual(output.status_code, 404) # Add a README to the git repo - First commit - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View first commit @@ -2203,9 +2203,10 @@ class PagureFlaskRepotests(tests.Modeltests): self.assertIn('

Project not found

', output.data) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View another commit @@ -2246,7 +2247,7 @@ class PagureFlaskRepotests(tests.Modeltests): self.session.add(item) self.session.commit() forkedgit = os.path.join( - self.path, 'forks', 'pingou', 'test3.git') + self.path, 'repos', 'forks', 'pingou', 'test3.git') tests.add_content_git_repo(forkedgit) tests.add_readme_git_repo(forkedgit) @@ -2311,14 +2312,14 @@ class PagureFlaskRepotests(tests.Modeltests): # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/c/bar.patch') self.assertEqual(output.status_code, 404) # Add a README to the git repo - First commit - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View first commit @@ -2350,9 +2351,10 @@ index 0000000..fb7093d self.assertTrue('Subject: Add a README file' in output.data) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View another commit @@ -2384,7 +2386,8 @@ index 0000000..11980b1 ) self.session.add(item) self.session.commit() - forkedgit = os.path.join(self.path, 'forks', 'pingou', 'test3.git') + forkedgit = os.path.join(self.path, 'repos', 'forks', 'pingou', + 'test3.git') tests.add_content_git_repo(forkedgit) tests.add_readme_git_repo(forkedgit) @@ -2436,7 +2439,7 @@ index 0000000..fb7093d # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/tree/') self.assertEqual(output.status_code, 200) @@ -2454,8 +2457,8 @@ index 0000000..fb7093d 'No content found in this repository' in output.data) # Add a README to the git repo - First commit - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) commit = repo.revparse_single('HEAD') # View first commit @@ -2491,7 +2494,8 @@ index 0000000..fb7093d ) self.session.add(item) self.session.commit() - forkedgit = os.path.join(self.path, 'forks', 'pingou', 'test3.git') + forkedgit = os.path.join(self.path, 'repos', 'forks', 'pingou', + 'test3.git') tests.add_content_git_repo(forkedgit) @@ -2534,7 +2538,7 @@ index 0000000..fb7093d user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/delete', follow_redirects=True) self.assertEqual(output.status_code, 404) @@ -2565,7 +2569,7 @@ index 0000000..fb7093d ) self.session.add(item) self.session.commit() - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git(os.path.join(self.path, 'docs')) output = self.app.post('/test/delete', follow_redirects=True) self.assertEqual(output.status_code, 404) @@ -2581,7 +2585,7 @@ index 0000000..fb7093d self.session.commit() # Create all the git repos - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git(os.path.join(self.path, 'docs')) tests.create_projects_git( os.path.join(self.path, 'tickets'), bare=True) @@ -2710,7 +2714,8 @@ index 0000000..fb7093d self.session.add(item) self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', + 'test3.git')) tests.add_content_git_repo( os.path.join(self.path, 'docs', 'pingou', 'test3.git')) tests.add_content_git_repo( @@ -2746,7 +2751,7 @@ index 0000000..fb7093d user = tests.FakeUser() with tests.user_set(pagure.APP, user): tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) # No project registered in the DB (no git repo) output = self.app.post('/foo/delete') @@ -2762,7 +2767,7 @@ index 0000000..fb7093d user = tests.FakeUser(username='pingou') with tests.user_set(pagure.APP, user): - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) ast.return_value = True output = self.app.post('/test/delete') @@ -2790,7 +2795,7 @@ index 0000000..fb7093d ) self.session.add(item) self.session.commit() - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/delete', follow_redirects=True) self.assertEqual(output.status_code, 200) @@ -2813,7 +2818,7 @@ index 0000000..fb7093d ) self.session.add(item) self.session.commit() - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git(os.path.join(self.path, 'docs')) output = self.app.post('/test/delete', follow_redirects=True) self.assertEqual(output.status_code, 200) @@ -2832,7 +2837,7 @@ index 0000000..fb7093d self.session.commit() # Create all the git repos - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git(os.path.join(self.path, 'docs')) tests.create_projects_git( os.path.join(self.path, 'tickets'), bare=True) @@ -2967,7 +2972,8 @@ index 0000000..fb7093d self.session.add(item) self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', + 'test3.git')) tests.add_content_git_repo( os.path.join(self.path, 'docs', 'pingou', 'test3.git')) tests.add_content_git_repo( @@ -3014,7 +3020,7 @@ index 0000000..fb7093d self.session.commit() # Create all the git repos - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git( os.path.join(self.path, 'docs'), bare=True) tests.create_projects_git( @@ -3094,7 +3100,7 @@ index 0000000..fb7093d self.session.commit() # Create all the git repos - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git( os.path.join(self.path, 'docs'), bare=True) tests.create_projects_git( @@ -3188,7 +3194,7 @@ index 0000000..fb7093d self.session.commit() # Create all the git repos - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) tests.create_projects_git( os.path.join(self.path, 'docs'), bare=True) tests.create_projects_git( @@ -3266,7 +3272,7 @@ index 0000000..fb7093d """ Test the new_repo_hook_token endpoint. """ ast.return_value = False tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) repo = pagure.get_authorized_project(self.session, 'test') self.assertEqual(repo.hook_token, 'aaabbbccc') @@ -3328,7 +3334,7 @@ index 0000000..fb7093d upgit.return_value = True sendmail.return_value = True tests.create_projects(self.session) - tests.create_projects_git(self.path) + tests.create_projects_git(os.path.join(self.path, 'repos')) user = tests.FakeUser() with tests.user_set(pagure.APP, user): @@ -3420,15 +3426,15 @@ index 0000000..fb7093d # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/releases') self.assertEqual(output.status_code, 200) self.assertIn('This project has not been tagged.', output.data) # Add a README to the git repo - First commit - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) - repo = pygit2.Repository(os.path.join(self.path, 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', 'test.git')) + repo = pygit2.Repository(os.path.join(self.path, 'repos', 'test.git')) first_commit = repo.revparse_single('HEAD') tagger = pygit2.Signature('Alice Doe', 'adoe@example.com', 12347, 0) repo.create_tag( @@ -3455,7 +3461,8 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), + bare=True) # No a repo admin output = self.app.get('/test/edit/foo/f/sources') @@ -3473,12 +3480,14 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) # Add some content to the git repo - tests.add_content_git_repo(os.path.join(self.path, 'test.git')) - tests.add_readme_git_repo(os.path.join(self.path, 'test.git')) + tests.add_content_git_repo(os.path.join(self.path, 'repos', + 'test.git')) + tests.add_readme_git_repo(os.path.join(self.path, 'repos', + 'test.git')) tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test.jpg') + os.path.join(self.path, 'repos', 'test.git'), 'test.jpg') tests.add_binary_git_repo( - os.path.join(self.path, 'test.git'), 'test_binary') + os.path.join(self.path, 'repos', 'test.git'), 'test_binary') output = self.app.get('/test/edit/master/foofile') self.assertEqual(output.status_code, 404) @@ -3556,9 +3565,7 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 200) self.assertIn( 'Commits - test - Pagure', output.data) - self.assertIn( - '\n Changes committed', - output.data) + self.assertIn('test commit', output.data) # Check file after the commit: output = self.app.get('/test/raw/master/f/sources') @@ -3578,11 +3585,11 @@ index 0000000..fb7093d self.session.commit() tests.add_content_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_readme_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git')) + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git')) tests.add_commit_git_repo( - os.path.join(self.path, 'forks', 'pingou', 'test3.git'), + os.path.join(self.path, 'repos', 'forks', 'pingou', 'test3.git'), ncommits=10) # Verify the nav links correctly when editing a file in a fork. @@ -3621,9 +3628,7 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 200) self.assertIn( 'Commits - test - Pagure', output.data) - self.assertIn( - '\n Changes committed', - output.data) + self.assertIn('test commit', output.data) # Check file after the commit: output = self.app.get('/test/raw/master/f/sources') @@ -3651,7 +3656,7 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - repos = tests.create_projects_git(self.path) + repos = tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/default/branch/') self.assertEqual(output.status_code, 403) @@ -3747,7 +3752,7 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - repo = tests.create_projects_git(self.path) + repo = tests.create_projects_git(os.path.join(self.path, 'repos')) output = self.app.post('/test/upload/') self.assertEqual(output.status_code, 403) @@ -3809,7 +3814,8 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), + bare=True) output = self.app.get('/test/token/new/') self.assertEqual(output.status_code, 403) @@ -3882,7 +3888,8 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), + bare=True) output = self.app.post('/test/token/revoke/123') self.assertEqual(output.status_code, 403) @@ -3958,7 +3965,7 @@ index 0000000..fb7093d self.assertEqual(output.status_code, 404) tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # User not logged in output = self.app.post('/test/b/master/delete') @@ -3986,7 +3993,7 @@ index 0000000..fb7093d self.assertIn('

Branch not found

', output.data) # Add a branch that we can delete - path = os.path.join(self.path, 'test.git') + path = os.path.join(self.path, 'repos', 'test.git') tests.add_content_git_repo(path) repo = pygit2.Repository(path) repo.create_branch('foo', repo.head.get_object()) @@ -4015,7 +4022,7 @@ index 0000000..fb7093d output.data) # Add a branch with a '/' in its name that we can delete - path = os.path.join(self.path, 'test.git') + path = os.path.join(self.path, 'repos', 'test.git') tests.add_content_git_repo(path) repo = pygit2.Repository(path) repo.create_branch('feature/foo', repo.head.get_object()) @@ -4056,7 +4063,7 @@ index 0000000..fb7093d # No git repo associated self.assertEqual(output.status_code, 404) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/docs/test/') self.assertEqual(output.status_code, 404) @@ -4064,7 +4071,7 @@ index 0000000..fb7093d def test_view_project_activity(self): """ Test the view_project_activity endpoint. """ tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) # Project Exists, but No DATAGREPPER_URL set output = self.app.get('/test/activity/') @@ -4086,7 +4093,7 @@ index 0000000..fb7093d def test_goimport(self): """ Test the go-import tag. """ tests.create_projects(self.session) - tests.create_projects_git(self.path, bare=True) + tests.create_projects_git(os.path.join(self.path, 'repos'), bare=True) output = self.app.get('/test/') self.assertEqual(output.status_code, 200) self.assertIn(' Date: May 22 2017 20:37:00 +0000 Subject: [PATCH 53/57] Update lib tests Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py index 08c15aa..36896aa 100644 --- a/tests/test_pagure_lib.py +++ b/tests/test_pagure_lib.py @@ -1261,7 +1261,7 @@ class PagureLibtests(tests.Modeltests): # Create a new project pagure.APP.config['GIT_FOLDER'] = gitfolder - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject', @@ -1275,7 +1275,12 @@ class PagureLibtests(tests.Modeltests): parent_id=None, ) self.session.commit() - self.assertEqual(msg, 'Project "testproject" created') + result = pagure.lib.tasks.get_result(tid).get() + self.assertEqual( + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': None}) # Try creating an existing project using a different case self.assertRaises( @@ -1354,7 +1359,7 @@ class PagureLibtests(tests.Modeltests): self.session.delete(repo) self.session.commit() - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject', @@ -1369,7 +1374,12 @@ class PagureLibtests(tests.Modeltests): ignore_existing_repo=True ) self.session.commit() - self.assertEqual(msg, 'Project "testproject" created') + result = pagure.lib.tasks.get_result(tid).get() + self.assertEqual( + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': None}) # Delete the repo from the DB so we can try again repo = pagure.lib._get_project(self.session, 'testproject') @@ -1383,9 +1393,7 @@ class PagureLibtests(tests.Modeltests): # Drop the main git repo and try again shutil.rmtree(gitrepo) - self.assertRaises( - pagure.exceptions.PagureException, - pagure.lib.new_project, + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject', @@ -1396,8 +1404,10 @@ class PagureLibtests(tests.Modeltests): ticketfolder=ticketfolder, requestfolder=requestfolder, description='description for testproject', - parent_id=None - ) + parent_id=None) + self.assertIn( + 'already exists', + str(pagure.lib.tasks.get_result(tid).get(propagate=False))) self.session.rollback() self.assertFalse(os.path.exists(gitrepo)) @@ -1452,7 +1462,7 @@ class PagureLibtests(tests.Modeltests): self.assertTrue(os.path.exists(requestrepo)) # Re-Try creating a 40 chars project this time allowing it - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='pingou/' + 's' * 40, @@ -1466,10 +1476,12 @@ class PagureLibtests(tests.Modeltests): parent_id=None, ) self.session.commit() + result = pagure.lib.tasks.get_result(tid).get() self.assertEqual( - msg, - 'Project "pingou/ssssssssssssssssssssssssssssssssssssssss" ' - 'created') + result, + {'endpoint': 'view_repo', + 'repo': 'pingou/ssssssssssssssssssssssssssssssssssssssss', + 'namespace': None}) def test_new_project_user_ns(self): """ Test the new_project of pagure.lib with user_ns on. """ @@ -1480,7 +1492,7 @@ class PagureLibtests(tests.Modeltests): # Create a new project with user_ns as True pagure.APP.config['GIT_FOLDER'] = gitfolder - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject', @@ -1495,7 +1507,12 @@ class PagureLibtests(tests.Modeltests): user_ns=True, ) self.session.commit() - self.assertEqual(msg, 'Project "pingou/testproject" created') + result = pagure.lib.tasks.get_result(tid).get() + self.assertEqual( + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': 'pingou'}) repo = pagure.lib._get_project( self.session, 'testproject', namespace='pingou') @@ -1512,7 +1529,7 @@ class PagureLibtests(tests.Modeltests): # Create a new project with a namespace and user_ns as True pagure.APP.config['GIT_FOLDER'] = gitfolder - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject2', @@ -1528,7 +1545,12 @@ class PagureLibtests(tests.Modeltests): user_ns=True, ) self.session.commit() - self.assertEqual(msg, 'Project "testns/testproject2" created') + result = pagure.lib.tasks.get_result(tid).get() + self.assertEqual( + result, + {'endpoint': 'view_repo', + 'repo': 'testproject2', + 'namespace': 'testns'}) repo = pagure.lib._get_project( self.session, 'testproject2', namespace='testns') @@ -2046,7 +2068,7 @@ class PagureLibtests(tests.Modeltests): self.assertEqual(len(projects), 0) # Create a new project - msg = pagure.lib.new_project( + tid = pagure.lib.new_project( session=self.session, user='pingou', name='testproject', @@ -2060,7 +2082,12 @@ class PagureLibtests(tests.Modeltests): parent_id=None, ) self.session.commit() - self.assertEqual(msg, 'Project "testproject" created') + result = pagure.lib.tasks.get_result(tid).get() + self.assertEqual( + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': None}) projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 1) @@ -2084,7 +2111,7 @@ class PagureLibtests(tests.Modeltests): # Fork - msg = pagure.lib.fork_project( + tid = pagure.lib.fork_project( session=self.session, user='foo', repo=project, @@ -2094,8 +2121,13 @@ class PagureLibtests(tests.Modeltests): requestfolder=requestfolder, ) self.session.commit() + result = pagure.lib.tasks.get_result(tid).get() self.assertEqual( - msg, 'Repo "testproject" cloned to "foo/testproject"') + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': None, + 'username': 'foo'}) projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 2) @@ -2108,14 +2140,12 @@ class PagureLibtests(tests.Modeltests): ['feature1', 'feature2', 'master'] ) - def test_fork_project_namespaced(self): """ Test the fork_project of pagure.lib on a namespaced project. """ gitfolder = os.path.join(self.path, 'repos') docfolder = os.path.join(self.path, 'docs') ticketfolder = os.path.join(self.path, 'tickets') requestfolder = os.path.join(self.path, 'requests') - pagure.APP.config['GIT_FOLDER'] = gitfolder projects = pagure.lib.search_projects(self.session) self.assertEqual(len(projects), 0) @@ -2178,17 +2208,16 @@ class PagureLibtests(tests.Modeltests): grepo = '%s.git' % os.path.join( docfolder, 'forks', 'foo', 'foonamespace', 'testproject') os.makedirs(grepo) - self.assertRaises( - pagure.exceptions.PagureException, - pagure.lib.fork_project, - session=self.session, - user='foo', - repo=repo, - gitfolder=gitfolder, - docfolder=docfolder, - ticketfolder=ticketfolder, - requestfolder=requestfolder, - ) + tid = pagure.lib.fork_project(session=self.session, + user='foo', + repo=repo, + gitfolder=gitfolder, + docfolder=docfolder, + ticketfolder=ticketfolder, + requestfolder=requestfolder) + self.assertIn( + 'already exists', + str(pagure.lib.tasks.get_result(tid).get(propagate=False))) self.session.rollback() shutil.rmtree(grepo) @@ -2196,17 +2225,17 @@ class PagureLibtests(tests.Modeltests): grepo = '%s.git' % os.path.join( ticketfolder, 'forks', 'foo', 'foonamespace', 'testproject') os.makedirs(grepo) - self.assertRaises( - pagure.exceptions.PagureException, - pagure.lib.fork_project, + tid = pagure.lib.fork_project( session=self.session, user='foo', repo=repo, gitfolder=gitfolder, docfolder=docfolder, ticketfolder=ticketfolder, - requestfolder=requestfolder, - ) + requestfolder=requestfolder) + self.assertIn( + 'already exists', + str(pagure.lib.tasks.get_result(tid).get(propagate=False))) self.session.rollback() shutil.rmtree(grepo) @@ -2214,26 +2243,23 @@ class PagureLibtests(tests.Modeltests): grepo = '%s.git' % os.path.join( requestfolder, 'forks', 'foo', 'foonamespace', 'testproject') os.makedirs(grepo) - self.assertRaises( - pagure.exceptions.PagureException, - pagure.lib.fork_project, + tid = pagure.lib.fork_project( session=self.session, user='foo', repo=repo, gitfolder=gitfolder, docfolder=docfolder, ticketfolder=ticketfolder, - requestfolder=requestfolder, - ) + requestfolder=requestfolder) + self.assertIn( + 'already exists', + str(pagure.lib.tasks.get_result(tid).get(propagate=False))) self.session.rollback() shutil.rmtree(grepo) - projects = pagure.lib.search_projects(self.session) - self.assertEqual(len(projects), 1) - # Fork worked - msg = pagure.lib.fork_project( + tid = pagure.lib.fork_project( session=self.session, user='foo', repo=repo, @@ -2243,19 +2269,19 @@ class PagureLibtests(tests.Modeltests): requestfolder=requestfolder, ) self.session.commit() + result = pagure.lib.tasks.get_result(tid).get() self.assertEqual( - msg, - 'Repo "foonamespace/testproject" cloned to ' - '"foo/foonamespace/testproject"') - - projects = pagure.lib.search_projects(self.session) - self.assertEqual(len(projects), 2) + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': 'foonamespace', + 'username': 'foo'}) # Fork a fork repo = pagure.lib._get_project(self.session, 'testproject', user='foo', namespace='foonamespace') - msg = pagure.lib.fork_project( + tid = pagure.lib.fork_project( session=self.session, user='pingou', repo=repo, @@ -2265,13 +2291,13 @@ class PagureLibtests(tests.Modeltests): requestfolder=requestfolder, ) self.session.commit() + result = pagure.lib.tasks.get_result(tid).get() self.assertEqual( - msg, - 'Repo "foonamespace/testproject" cloned to ' - '"pingou/foonamespace/testproject"') - - projects = pagure.lib.search_projects(self.session) - self.assertEqual(len(projects), 3) + result, + {'endpoint': 'view_repo', + 'repo': 'testproject', + 'namespace': 'foonamespace', + 'username': 'pingou'}) @patch('pagure.lib.notify.send_email') def test_new_pull_request(self, mockemail): From de6f339c2f9e1b063c93f59139ac265ad94191fc Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 23 2017 09:23:49 +0000 Subject: [PATCH 54/57] Use the most compatible way of arguments for celery Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index 841e503..fa9486d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -190,7 +190,7 @@ class Modeltests(unittest.TestCase): # Using eventlet so that worker.terminate kills everything self.worker = subprocess.Popen( ['/usr/bin/celery', '-A', 'pagure.lib.tasks', 'worker', - '--loglevel', 'info', '--concurrency', '2', '--pool', 'eventlet', + '--loglevel=info', '--concurrency=2', '--pool=eventlet', '--without-gossip', '--without-mingle', '--quiet'], env={'PAGURE_BROKER_URL': celery_broker_url, 'PAGURE_CONFIG': os.path.join(self.path, 'config'), From 0bfbed9c0a0091b05cfe16cb76dbc42a67a11cd9 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 23 2017 09:45:42 +0000 Subject: [PATCH 55/57] Fix task ready check Signed-off-by: Patrick Uiterwijk --- diff --git a/pagure/ui/app.py b/pagure/ui/app.py index 2a66165..54eefad 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -490,7 +490,7 @@ def new_project(): @APP.route('/wait/') def wait_task(taskid): status = pagure.lib.tasks.get_result(taskid) - if status.ready: + if status.ready(): result = status.get(timeout=0, propagate=False) if status.failed(): flask.flash('Your task failed: %s' % str(result)) From f43a2e05fb56ae58047c40b78e943102569e688f Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: May 23 2017 09:53:59 +0000 Subject: [PATCH 56/57] Make the test suites quiet again Signed-off-by: Patrick Uiterwijk --- diff --git a/tests/__init__.py b/tests/__init__.py index fa9486d..b7fe8be 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -188,6 +188,7 @@ class Modeltests(unittest.TestCase): # Start a worker # Using cocurrency 2 to test with some concurrency, but not be heavy # Using eventlet so that worker.terminate kills everything + self.workerlog = open(os.path.join(self.path, 'worker.log'), 'w') self.worker = subprocess.Popen( ['/usr/bin/celery', '-A', 'pagure.lib.tasks', 'worker', '--loglevel=info', '--concurrency=2', '--pool=eventlet', @@ -196,7 +197,9 @@ class Modeltests(unittest.TestCase): 'PAGURE_CONFIG': os.path.join(self.path, 'config'), 'PYTHONPATH': '.'}, cwd=os.path.normpath(os.path.join(os.path.dirname(__file__), - '..'))) + '..')), + stdout=self.workerlog, + stderr=self.workerlog) self.worker.poll() if self.worker.returncode is not None: raise Exception('Worker failed to start') From 7833e9ecd73430c3744e142e6e9fa8c831d3a254 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: May 23 2017 09:54:38 +0000 Subject: [PATCH 57/57] Make the wait page a little prettier Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/waiting.html b/pagure/templates/waiting.html index 2a71ea2..cb8c221 100644 --- a/pagure/templates/waiting.html +++ b/pagure/templates/waiting.html @@ -8,13 +8,25 @@ {% block content %} -

- Waiting -

-

- We are waiting for your task to finish. - This page should be refreshed automatically, but if not click Here -

+
+
+
+
+
+ Waiting +
+
+

+ We are waiting for your task to finish. + This page should be refreshed automatically, but if not click + Here +

+
+
+
+
+
+ {% endblock %}