From f65cc06675847f395075fa51e8cbaa5a43c59e25 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 1/18] Write the ssh key to disk rather than relying on /dev/stdin This is required to get the tests to pass in a container where /dev/stdin is not available inside the process running the tests. So rather than relying on /dev/stdin, we just write the key to disk in a temp folder that is cleaned up automatically once we're done checking if the ssh key is valid or not. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/query.py b/pagure/lib/query.py index 80d94a9..69ded51 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ - (c) 2014-2018 - Copyright Red Hat Inc + (c) 2014-2019 - Copyright Red Hat Inc Authors: Pierre-Yves Chibon @@ -30,6 +30,7 @@ import hashlib import logging import os import tempfile +import shutil import subprocess import uuid import markdown @@ -208,15 +209,19 @@ def is_valid_ssh_key(key, fp_hash="SHA256"): key = key.strip() if not key: return None - with tempfile.TemporaryFile() as f: - f.write(key.encode("utf-8")) - f.seek(0) - cmd = ["/usr/bin/ssh-keygen", "-l", "-f", "/dev/stdin", "-E", fp_hash] - proc = subprocess.Popen( - cmd, stdin=f, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) + tmpdirname = tempfile.mkdtemp() + filename = os.path.join(tmpdirname, "key") + with open(filename, "w") as stream: + stream.write(key) + cmd = ["/usr/bin/ssh-keygen", "-l", "-f", filename, "-E", fp_hash] + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, stderr = proc.communicate() + shutil.rmtree(tmpdirname) if proc.returncode != 0: + _log.warning("STDOUT: %s", stdout) + _log.warning("STDERR: %s", stderr) return False stdout = stdout.decode("utf-8") From 921afa5f4d5e5f6132456fbae0f7974c277c60f9 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 2/18] Move all the container definition files from a docker folder to containers After all these files are not docker specific and will in fact work for every container engine (including podman which we're testing with). We're adjusting the docker-compose file to point to the right folder so it keeps working, and the .gitignore file. Signed-off-by: Pierre-Yves Chibon --- diff --git a/.gitignore b/.gitignore index 2e0d099..0bb1b03 100644 --- a/.gitignore +++ b/.gitignore @@ -48,8 +48,8 @@ Vagrantfile *.retry .vagrant -#Ignore generated dockerfile -dev/docker/test_env +# Ignore generated dockerfile +dev/containers/test_env # Tox virtualenvs .tox diff --git a/dev/containers/centos7-rpms b/dev/containers/centos7-rpms new file mode 100644 index 0000000..a23d78e --- /dev/null +++ b/dev/containers/centos7-rpms @@ -0,0 +1,32 @@ +FROM centos:7 + +RUN yum -y install \ + epel-release \ + yum-utils \ + rpm-build \ + python-setuptools \ + python-coverage \ + python-nose \ + python2-mock \ + python-docutils \ + python-enum \ + python-flask \ + redis \ + which +ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo + + +RUN cd /pagure && \ + yum install -y `grep "Requires:" /pagure/files/pagure.spec | \ + awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ + sed -e "s|%{python_pkgversion}||"` && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 + +#RUN yum -y install python2-mock redis +# yum clean all &&\ + +#ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +WORKDIR /pagure +ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f"] +#ENTRYPOINT ["py.test", "-x", "tests/"] +CMD [] diff --git a/dev/containers/ev b/dev/containers/ev new file mode 100644 index 0000000..005ee26 --- /dev/null +++ b/dev/containers/ev @@ -0,0 +1,22 @@ +FROM registry.fedoraproject.org/fedora:28 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-markdown python-psutil \ + python-pygit2 python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 python-trololio \ + python-celery + +WORKDIR /code +ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code diff --git a/dev/containers/fedora-infra-tags.repo b/dev/containers/fedora-infra-tags.repo new file mode 100644 index 0000000..4bd9b21 --- /dev/null +++ b/dev/containers/fedora-infra-tags.repo @@ -0,0 +1,6 @@ +[infrastructure-tags] +name=Fedora Infrastructure tag epel7-infra - x86_64 +baseurl=https://kojipkgs.fedoraproject.org/repos-dist/epel7-infra/latest/x86_64/ +enabled=1 +gpgcheck=1 +gpgkey=https://infrastructure.fedoraproject.org/repo/infra/RPM-GPG-KEY-INFRA-TAGS diff --git a/dev/containers/logcom b/dev/containers/logcom new file mode 100644 index 0000000..d46512d --- /dev/null +++ b/dev/containers/logcom @@ -0,0 +1,28 @@ +FROM registry.fedoraproject.org/fedora:28 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ + python3-arrow python3-binaryornot python3-bleach python3-blinker \ + python3-chardet python3-cryptography python3-docutils python3-flask \ + python3-flask-wtf python3-markdown python3-psutil \ + python3-pygit2 python3-fedora python3-openid python3-openid-cla \ + python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ + python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ + python3-filelock \ + python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ + python3-kitchen + +RUN dnf install -y python3-celery + +WORKDIR /code +ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks_services", "worker", "--loglevel", "info", "-Q", "pagure_logcom"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code +VOLUME ["/attachments"] +# Ideally this would run as non-root, but that needs the /repos owned correctly +ENV C_FORCE_ROOT true diff --git a/dev/containers/test_env_template b/dev/containers/test_env_template new file mode 100644 index 0000000..dc5869b --- /dev/null +++ b/dev/containers/test_env_template @@ -0,0 +1,15 @@ +FROM $base_image + +$epel_pkg +$infra_repo + +RUN $pkg_mgr install -y $pkg_list \ + && $pkg_mgr clean all \ +# for the hooks we need to have /usr/bin/python + && cp /usr/bin/python3 /usr/bin/python \ +# for the alembic test we need /usr/bin/alembic + && cp /usr/bin/alembic-3 /usr/bin/alembic + +WORKDIR /pagure +ENTRYPOINT [ "/pagure/runtests.py", "run" ] +CMD [] diff --git a/dev/containers/web b/dev/containers/web new file mode 100644 index 0000000..78594b0 --- /dev/null +++ b/dev/containers/web @@ -0,0 +1,26 @@ +FROM registry.fedoraproject.org/fedora:28 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python2-devel python-setuptools python-nose python2-bcrypt python-alembic \ + python-arrow python-binaryornot python-bleach python-blinker \ + python-chardet python-cryptography python-docutils python-flask \ + python-flask-wtf python-markdown python-psutil \ + python-pygit2 python-fedora python-openid python-openid-cla \ + python-openid-teams python-straight-plugin python-wtforms python-munch \ + python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ + python-fedora-flask python2-pillow python2-psycopg2 python2-celery \ + findutils +COPY web-run /run.sh + +WORKDIR /code +# Openshift: --no-debug +ENTRYPOINT ["/usr/bin/bash", "/run.sh"] +EXPOSE 5000 + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code +VOLUME ["/attachments"] diff --git a/dev/containers/web-run b/dev/containers/web-run new file mode 100644 index 0000000..15d8668 --- /dev/null +++ b/dev/containers/web-run @@ -0,0 +1,14 @@ +#!/bin/bash -xe +pip install -r requirements-testing.txt +python setup.py build +if [ ! -f /attachments/inited ]; +then + echo "Giving Postgres time to start" + sleep 10 + touch /attachments/inited + PAGURE_CONFIG=/code/dev/openshift.cfg python createdb.py --initial /code/dev/openshift_alembic.ini +else + alembic --config /code/dev/openshift_alembic.ini upgrade head +fi + +exec /usr/bin/python /code/runserver.py --host 0.0.0.0 --config /code/dev/openshift.cfg diff --git a/dev/containers/worker b/dev/containers/worker new file mode 100644 index 0000000..c6d6ee7 --- /dev/null +++ b/dev/containers/worker @@ -0,0 +1,28 @@ +FROM registry.fedoraproject.org/fedora:28 +MAINTAINER Patrick Uiterwijk + +VOLUME ["/repos"] +RUN mkdir /code + +RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ + python3-arrow python3-binaryornot python3-bleach python3-blinker \ + python3-chardet python3-cryptography python3-docutils python3-flask \ + python3-flask-wtf python3-markdown python3-psutil \ + python3-pygit2 python3-fedora python3-openid python3-openid-cla \ + python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ + python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ + python3-filelock python3-bleach python3-cryptography \ + python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ + python3-blinker + +RUN dnf install -y python3-celery + +WORKDIR /code +ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info"] + +# Code injection is last to make optimal use of caches +VOLUME ["/code"] +# Openshift: COPY / /code +VOLUME ["/attachments"] +# Ideally this would run as non-root, but that needs the /repos owned correctly +ENV C_FORCE_ROOT true diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index b04893e..cc0464b 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -4,7 +4,7 @@ volumes: services: web: build: - context: ./docker + context: ./containers dockerfile: web depends_on: - redis @@ -22,7 +22,7 @@ services: - PAGURE_CONFIG=/code/dev/openshift.cfg worker: build: - context: ./docker + context: ./containers dockerfile: worker depends_on: - redis @@ -37,7 +37,7 @@ services: - PAGURE_CONFIG=/code/dev/openshift.cfg logcom: build: - context: ./docker + context: ./containers dockerfile: logcom depends_on: - redis @@ -52,7 +52,7 @@ services: - PAGURE_CONFIG=/code/dev/openshift.cfg ev: build: - context: ./docker + context: ./containers dockerfile: ev depends_on: - redis diff --git a/dev/docker/ev b/dev/docker/ev deleted file mode 100644 index 005ee26..0000000 --- a/dev/docker/ev +++ /dev/null @@ -1,22 +0,0 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-markdown python-psutil \ - python-pygit2 python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python-trololio \ - python-celery - -WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code diff --git a/dev/docker/fedora-infra-tags.repo b/dev/docker/fedora-infra-tags.repo deleted file mode 100644 index 4bd9b21..0000000 --- a/dev/docker/fedora-infra-tags.repo +++ /dev/null @@ -1,6 +0,0 @@ -[infrastructure-tags] -name=Fedora Infrastructure tag epel7-infra - x86_64 -baseurl=https://kojipkgs.fedoraproject.org/repos-dist/epel7-infra/latest/x86_64/ -enabled=1 -gpgcheck=1 -gpgkey=https://infrastructure.fedoraproject.org/repo/infra/RPM-GPG-KEY-INFRA-TAGS diff --git a/dev/docker/logcom b/dev/docker/logcom deleted file mode 100644 index d46512d..0000000 --- a/dev/docker/logcom +++ /dev/null @@ -1,28 +0,0 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ - python3-arrow python3-binaryornot python3-bleach python3-blinker \ - python3-chardet python3-cryptography python3-docutils python3-flask \ - python3-flask-wtf python3-markdown python3-psutil \ - python3-pygit2 python3-fedora python3-openid python3-openid-cla \ - python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ - python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ - python3-filelock \ - python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ - python3-kitchen - -RUN dnf install -y python3-celery - -WORKDIR /code -ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks_services", "worker", "--loglevel", "info", "-Q", "pagure_logcom"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code -VOLUME ["/attachments"] -# Ideally this would run as non-root, but that needs the /repos owned correctly -ENV C_FORCE_ROOT true diff --git a/dev/docker/test_env_template b/dev/docker/test_env_template deleted file mode 100644 index dc5869b..0000000 --- a/dev/docker/test_env_template +++ /dev/null @@ -1,15 +0,0 @@ -FROM $base_image - -$epel_pkg -$infra_repo - -RUN $pkg_mgr install -y $pkg_list \ - && $pkg_mgr clean all \ -# for the hooks we need to have /usr/bin/python - && cp /usr/bin/python3 /usr/bin/python \ -# for the alembic test we need /usr/bin/alembic - && cp /usr/bin/alembic-3 /usr/bin/alembic - -WORKDIR /pagure -ENTRYPOINT [ "/pagure/runtests.py", "run" ] -CMD [] diff --git a/dev/docker/web b/dev/docker/web deleted file mode 100644 index 78594b0..0000000 --- a/dev/docker/web +++ /dev/null @@ -1,26 +0,0 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose python2-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-markdown python-psutil \ - python-pygit2 python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python2-celery \ - findutils -COPY web-run /run.sh - -WORKDIR /code -# Openshift: --no-debug -ENTRYPOINT ["/usr/bin/bash", "/run.sh"] -EXPOSE 5000 - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code -VOLUME ["/attachments"] diff --git a/dev/docker/web-run b/dev/docker/web-run deleted file mode 100644 index 15d8668..0000000 --- a/dev/docker/web-run +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -xe -pip install -r requirements-testing.txt -python setup.py build -if [ ! -f /attachments/inited ]; -then - echo "Giving Postgres time to start" - sleep 10 - touch /attachments/inited - PAGURE_CONFIG=/code/dev/openshift.cfg python createdb.py --initial /code/dev/openshift_alembic.ini -else - alembic --config /code/dev/openshift_alembic.ini upgrade head -fi - -exec /usr/bin/python /code/runserver.py --host 0.0.0.0 --config /code/dev/openshift.cfg diff --git a/dev/docker/worker b/dev/docker/worker deleted file mode 100644 index c6d6ee7..0000000 --- a/dev/docker/worker +++ /dev/null @@ -1,28 +0,0 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk - -VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ - python3-arrow python3-binaryornot python3-bleach python3-blinker \ - python3-chardet python3-cryptography python3-docutils python3-flask \ - python3-flask-wtf python3-markdown python3-psutil \ - python3-pygit2 python3-fedora python3-openid python3-openid-cla \ - python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ - python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ - python3-filelock python3-bleach python3-cryptography \ - python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ - python3-blinker - -RUN dnf install -y python3-celery - -WORKDIR /code -ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info"] - -# Code injection is last to make optimal use of caches -VOLUME ["/code"] -# Openshift: COPY / /code -VOLUME ["/attachments"] -# Ideally this would run as non-root, but that needs the /repos owned correctly -ENV C_FORCE_ROOT true From 58a0c3e98b9945a612cb6b5fe9684db20b70be74 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 3/18] Keep pagure working on older version of flask not having iter_blueprints This function has been added after 0.10 and is thus not available in EPEL7 at this time. This commit makes pagure compatible with this version of flask by replacing the call to iter_blueprints() by using the attribute .blueprints which is a dict of all the blueprints of the application. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/templates/settings.html b/pagure/templates/settings.html index a9f9ad4..1e25547 100644 --- a/pagure/templates/settings.html +++ b/pagure/templates/settings.html @@ -102,8 +102,8 @@ href="#deleteproject-tab" role="tab" aria-controls="deleteproject">Delete Project {% endif %} - {% for blueprint in main_app.iter_blueprints() %} - {% if blueprint.name not in [ + {% for blueprint in main_app.blueprints %} + {% if blueprint not in [ 'api_ns', 'ui_ns', 'internal_ns', 'theme'] %} Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 4/18] Make the runtest script adjust a couple of files for py3/py2 differences This makes sure we are using py3 or py2 consistently in the tests when checking only one of these. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runtests.py b/runtests.py index 7e25ee6..cb55090 100755 --- a/runtests.py +++ b/runtests.py @@ -31,6 +31,7 @@ NUMPROCS = multiprocessing.cpu_count() - 1 if os.environ.get('BUILD_ID'): NUMPROCS = multiprocessing.cpu_count() +HERE = os.path.join(os.path.dirname(os.path.abspath(__file__))) LOG = logging.getLogger(__name__) def setup_parser(): @@ -310,7 +311,7 @@ def do_run(args): try: with open(failed_tests_fullpath, "r") as ffile: suites = json.loads(ffile.read()) - except json.decoder.JSONDecodeError: + except: bname = os.path.basename(args.failed_tests) if bname.endswith(".py") and bname.startswith("test_"): suites.append(bname.replace(".py", "")) @@ -422,6 +423,26 @@ def _run_test_suites(args, suites): if not pyvers: return 1 + if len(pyvers) == 1: + if pyvers[0] == 2: + subprocess.check_call([ + "sed", "-i", "-e", "s|python|python2|", + "pagure/hooks/files/hookrunner" + ]) + subprocess.check_call([ + "sed", "-i", "-e", "s|['alembic',|['alembic-2',|", + "tests/test_alembic.py" + ]) + elif pyvers[0] == 3: + subprocess.check_call([ + "sed", "-i", "-e", "s|python|python3|", + "pagure/hooks/files/hookrunner" + ], cwd=HERE) + subprocess.check_call([ + "sed", "-i", "-e", "s|\['alembic',|\['alembic-3',|", + "tests/test_alembic.py" + ], cwd=HERE) + for suite in suites: for pyver in pyvers: NUMREMAINING += 1 @@ -443,6 +464,13 @@ def _run_test_suites(args, suites): print() print("All work done") + subprocess.check_call([ + "git", + "checkout", + "pagure/hooks/files/hookrunner", + "tests/test_alembic.py" + ]) + # Gather results print() print() From 8d17325fbe1b9e7a4b599cd486498b32621af917 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 5/18] Improve the centos7-rpms-py2 container These changes make it possible to run the tests in this environment Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/containers/centos7-rpms b/dev/containers/centos7-rpms index a23d78e..8ed9083 100644 --- a/dev/containers/centos7-rpms +++ b/dev/containers/centos7-rpms @@ -1,32 +1,40 @@ FROM centos:7 RUN yum -y install \ - epel-release \ + epel-release + +RUN yum -y install \ yum-utils \ rpm-build \ python-setuptools \ python-coverage \ python-nose \ - python2-mock \ + python-mock \ python-docutils \ python-enum \ python-flask \ + python2-flake8 \ redis \ which -ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo - RUN cd /pagure && \ - yum install -y `grep "Requires:" /pagure/files/pagure.spec | \ + yum install --enablerepo=epel-testing -y \ + `grep "Requires:" /pagure/files/pagure.spec | \ awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ sed -e "s|%{python_pkgversion}||"` && \ + yum clean all && \ localedef -i en_US -f UTF-8 en_US.UTF-8 -#RUN yum -y install python2-mock redis -# yum clean all &&\ +RUN \ + sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \ + sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \ + sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \ + /pagure/requirements.txt && \ + cd /pagure && python setup.py build + #ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 WORKDIR /pagure -ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f"] +ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f", "--py2"] #ENTRYPOINT ["py.test", "-x", "tests/"] CMD [] From b8350d960284010be94159c4e2a0afebdae4665b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 6/18] Re-organize the imports Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/query.py b/pagure/lib/query.py index 69ded51..af39174 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -23,27 +23,28 @@ try: except ImportError: # pragma: no cover import json +import copy import datetime import fnmatch import functools import hashlib import logging import os -import tempfile import shutil import subprocess +import tempfile import uuid -import markdown -import werkzeug + from collections import Counter from math import ceil -import copy import bleach +import markdown import redis import six import sqlalchemy import sqlalchemy.schema +import werkzeug from six.moves.urllib_parse import urlparse, urlencode, parse_qsl from sqlalchemy import func From 19ab9004b219100d843c141b220ccdff57236120 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 7/18] Apparently in a CentOS7 container, it wants a fresh session more often Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/__init__.py b/tests/__init__.py index d27badf..87ecac2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -502,6 +502,9 @@ class Modeltests(SimplePagureTest): self.app.get = create_maybe_waiter(self.app.get, self.app.get) self.app.post = create_maybe_waiter(self.app.post, self.app.get) + # Refresh the DB session + self.session = pagure.lib.query.create_session(self.dbpath) + def tearDown(self): # pylint: disable=invalid-name """ Remove the test.db database if there is one. """ tests_state["broker_client"].flushall() From 07007f2e6250f68217200e0c6966e35983487ac2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 8/18] Be explicit about the unicode string used as filename This allows the tests to pass on a CentOS7 container Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/__init__.py b/tests/__init__.py index 87ecac2..44479b0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -779,9 +779,9 @@ def add_content_git_repo(folder, branch='master', append=None): with open(os.path.join(newfolder, subfolder, 'file'), 'w') as stream: stream.write('foo\n bar\nbaz') repo.index.add(os.path.join(subfolder, 'file')) - with open(os.path.join(newfolder, subfolder, 'fileŠ'), 'w') as stream: + with open(os.path.join(newfolder, subfolder, u'fileŠ'), 'w') as stream: stream.write('foo\n bar\nbaz') - repo.index.add(os.path.join(subfolder, 'fileŠ')) + repo.index.add(os.path.join(subfolder, u'fileŠ')) repo.index.write() # Commits the files added From b3d8b55eca4188e5da72ea2b81eac37ec96d77cb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 9/18] Refresh the DB session at the end of the setUp in all cases Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/__init__.py b/tests/__init__.py index 44479b0..c81e383 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -409,6 +409,9 @@ class SimplePagureTest(unittest.TestCase): gr_mock = self.gr_patcher.start() gr_mock.side_effect = lambda tid: tests_state["results"][tid] + # Refresh the DB session + self.session = pagure.lib.query.create_session(self.dbpath) + def tearDown(self): self.gr_patcher.stop() self.session.rollback() From c716ab626781a9861832ff4b56492746597313bf Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 10/18] Adjust the unit-tests for old markdown Looks like the html generated between old markdown library and newer one changed, so this commit adjusts our tests so they pass on the old markdown version. Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py index cbb81ad..2d6d535 100644 --- a/tests/test_pagure_lib.py +++ b/tests/test_pagure_lib.py @@ -4118,6 +4118,11 @@ class PagureLibtests(tests.Modeltests): pass bleach_v[idx] = val + # old markdown generate other html + import markdown + markdown_v = markdown.__version__.version_info + old_markdown = markdown_v < (2, 6, 0) + texts = [ 'foo bar test#1 see?', 'foo bar pingou/test#2 I mean, really', @@ -4245,18 +4250,35 @@ class PagureLibtests(tests.Modeltests): '

@pingou at start of line

', # 'but not someone@pingou.com', '

but not someone@pingou.com

', - # '[![Fedora_infinity_small.png]' - # '(/test/issue/raw/Fedora_infinity_small.png)]' - # '(/test/issue/raw/Fedora_infinity_small.png)', - '

' - 'Fedora_infinity_small.png' - '

', ] + if old_markdown: + expected.append( + # '[![Fedora_infinity_small.png]' + # '(/test/issue/raw/Fedora_infinity_small.png)]' + # '(/test/issue/raw/Fedora_infinity_small.png)', + '

' + 'Fedora_infinity_small.png' + '

' + ) + else: + expected.append( + # '[![Fedora_infinity_small.png]' + # '(/test/issue/raw/Fedora_infinity_small.png)]' + # '(/test/issue/raw/Fedora_infinity_small.png)', + '

' + 'Fedora_infinity_small.png' + '

' + ) + with self.app.application.app_context(): g.session = self.session for idx, text in enumerate(texts): From 671c7992070a59015deab8cb8917886e050cb661 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 11/18] Skip running flake8 on old flake8 version It returns too many false-positive Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_style.py b/tests/test_style.py index 31e5fa3..6af9509 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -38,6 +38,21 @@ class TestStyle(unittest.TestCase): sys.executable, '-m', 'flake8', '--ignore=E712,W503,E203', REPO_PATH ] + + # check if we have an old flake8 or not + import flake8 + flake8_v = flake8.__version__.split('.') + for idx, val in enumerate(flake8_v): + try: + val = int(val) + except ValueError: + pass + flake8_v[idx] = val + old_flake = tuple(flake8_v) < (3, 0) + + if old_flake: + raise unittest.SkipTest("Flake8 version too old to be useful") + proc = subprocess.Popen( flake8_command, stdout=subprocess.PIPE, From 8645c3933e173bf91dbb52c3175b3b5ef6bfa271 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 12/18] Make the SECRET_KEY a str() all the time This is needed to get pagure to work on old systems/version of wtforms. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/default_config.py b/pagure/default_config.py index 481c05c..6872db3 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -20,7 +20,7 @@ from pagure.mail_logging import ContextInjector, MSG_FORMAT ADMIN_SESSION_LIFETIME = timedelta(minutes=20) # secret key used to generate unique csrf token -SECRET_KEY = "" +SECRET_KEY = str("") # url to the database server: DB_URL = "sqlite:////var/tmp/pagure_dev.sqlite" From b562bcec6901cef8ab724d7bee6927d40509ff21 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 13/18] Fix the request context as data is only accessible in POST requests This worked on new flask version and doesn't on old version but this commit makes the test works on old and new flask versions. Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_flask_api.py b/tests/test_pagure_flask_api.py index 2a3e81e..3732f12 100644 --- a/tests/test_pagure_flask_api.py +++ b/tests/test_pagure_flask_api.py @@ -58,7 +58,8 @@ class PagureFlaskApitests(tests.SimplePagureTest): # test_request_context doesn't set flask.g, but some teardown # functions try to use that, so let's exclude them self._app.teardown_request_funcs = {} - with self._app.test_request_context('/api/0/version', data=data): + with self._app.test_request_context( + '/api/0/version', method="POST", data=data): self.assertEqual(pagure.api.get_request_data()['foo'], 'bar') data = json.dumps(data) with self._app.test_request_context('/api/0/version', data=data, From dddc2b14391f971a093f384c84a9c30ab10a4c9a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 05 2019 15:53:49 +0000 Subject: [PATCH 14/18] Fix the pagure_lib_git_auth test on old system Basically, rather than checking the entire string we split it into two tests so that the weird character added on old system doesn't break the test. Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_pagure_lib_git_auth.py b/tests/test_pagure_lib_git_auth.py index 3488006..e12178d 100644 --- a/tests/test_pagure_lib_git_auth.py +++ b/tests/test_pagure_lib_git_auth.py @@ -105,10 +105,10 @@ class PagureLibGitAuthtests(tests.Modeltests): output_text = output.get_data(as_text=True) self.assertIn( "Remote hook declined the push: " - "Denied push for ref 'refs/heads/master' for user 'pingou'\n" - "All changes have been rejected", + "Denied push for ref 'refs/heads/master' for user 'pingou'", output_text ) + self.assertIn("All changes have been rejected", output_text) # Check file after the commit: output = self.app.get('/hooktest/raw/master/f/sources') @@ -144,10 +144,10 @@ class PagureLibGitAuthtests(tests.Modeltests): output_text = output.get_data(as_text=True) self.assertIn( "Remote hook declined the push: " - "Denied push for ref 'refs/heads/master' for user 'pingou'\n" - "All changes have been rejected", + "Denied push for ref 'refs/heads/master' for user 'pingou'", output_text ) + self.assertIn("All changes have been rejected", output_text) # Change something in the "source" branch data = { From 479e0a39574b0efebd18294f60b43be177f5acd5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 07 2019 14:41:35 +0000 Subject: [PATCH 15/18] Provide a way to run the tests in containers using RPMs on F29 and C7 Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/containers/centos7-rpms b/dev/containers/centos7-rpms deleted file mode 100644 index 8ed9083..0000000 --- a/dev/containers/centos7-rpms +++ /dev/null @@ -1,40 +0,0 @@ -FROM centos:7 - -RUN yum -y install \ - epel-release - -RUN yum -y install \ - yum-utils \ - rpm-build \ - python-setuptools \ - python-coverage \ - python-nose \ - python-mock \ - python-docutils \ - python-enum \ - python-flask \ - python2-flake8 \ - redis \ - which - -RUN cd /pagure && \ - yum install --enablerepo=epel-testing -y \ - `grep "Requires:" /pagure/files/pagure.spec | \ - awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ - sed -e "s|%{python_pkgversion}||"` && \ - yum clean all && \ - localedef -i en_US -f UTF-8 en_US.UTF-8 - -RUN \ - sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \ - sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \ - sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \ - /pagure/requirements.txt && \ - cd /pagure && python setup.py build - - -#ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 -WORKDIR /pagure -ENTRYPOINT ["python2", "/pagure/runtests.py", "run", "-f", "--py2"] -#ENTRYPOINT ["py.test", "-x", "tests/"] -CMD [] diff --git a/dev/containers/centos7-rpms-py2 b/dev/containers/centos7-rpms-py2 new file mode 100644 index 0000000..26ee266 --- /dev/null +++ b/dev/containers/centos7-rpms-py2 @@ -0,0 +1,53 @@ +FROM centos:7 + +RUN yum -y install \ + epel-release + +ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo + +RUN yum -y install \ + yum-utils \ + rpm-build \ + python-setuptools \ + python-coverage \ + python-nose \ + python-mock \ + python-docutils \ + python-enum \ + python-flask \ + python-flake8 \ + redis \ + which \ + git \ + repoSpanner \ + repoSpanner-bridge + +RUN yum install -y https://kojipkgs.fedoraproject.org//work/tasks/5657/32535657/python-werkzeug-0.9.6-1.el7.noarch.rpm + +RUN cd / \ + && git clone https://pagure.io/pagure.git \ + && chmod +x /pagure/dev/containers/runtests_py2.sh + +# Install all the requirements from the spec file and replace the macro +# %{python_pkgversion} by empty string which thus installs all the py2 +# version of the dependencies. +RUN cd /pagure && \ + yum install --enablerepo=epel-testing -y \ + `rpmspec -q --requires /pagure/files/pagure.spec | \ + sed -e "s|%{python_pkgversion}||"` && \ + yum clean all && \ + localedef -i en_US -f UTF-8 en_US.UTF-8 + +# The old version of setuptools in CentOS7 does not support restrictions on +# the requirements file, so drop them +RUN \ + sed -i -e "s|;python_version<\"3.4\"||" /pagure/requirements.txt && \ + sed -i -e "s|;python_version<=\"2.7\"||" /pagure/requirements.txt && \ + sed -i -e "s|python3-openid;python_version>=\"3.0\"||" \ + /pagure/requirements.txt && \ + cd /pagure && python setup.py build + + +WORKDIR /pagure +ENTRYPOINT ["/pagure/dev/containers/runtests_py2.sh"] +CMD [] diff --git a/dev/containers/f29-rpms-py3 b/dev/containers/f29-rpms-py3 new file mode 100644 index 0000000..bb000a7 --- /dev/null +++ b/dev/containers/f29-rpms-py3 @@ -0,0 +1,31 @@ +FROM fedora:29 + +RUN dnf -y install \ + python3-setuptools \ + python3-coverage \ + python3-nose \ + python3-mock \ + python3-docutils \ + python3-black \ + python3-flake8 \ + redis \ + which \ + git + +RUN cd / \ + && git clone https://pagure.io/pagure.git \ + && chmod +x /pagure/dev/containers/runtests_py3.sh + +# Install all the requirements from the spec file and replace the macro +# %{python_pkgversion} by '3' which thus installs all the py3 version of +# the dependencies. +RUN dnf install -y `grep "Requires:" /pagure/files/pagure.spec | \ + awk '{split($0, a, " "); print a[2]}' |grep -v "%{name}" | \ + sed -e "s|%{python_pkgversion}|3|"` && \ + dnf clean all && \ + cd /pagure && python setup.py build + + +WORKDIR /pagure +ENTRYPOINT ["/pagure/dev/containers/runtests_py3.sh"] +CMD [] diff --git a/dev/containers/runtests_py2.sh b/dev/containers/runtests_py2.sh new file mode 100644 index 0000000..b161b17 --- /dev/null +++ b/dev/containers/runtests_py2.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +ls -l / + +echo "============== ENVIRONMENT =============" +/usr/bin/env +echo "============== END ENVIRONMENT =============" + +if [ -n "$REPO" -a -n "$BRANCH" ]; then +git remote rm proposed || true +git gc --auto +git remote add proposed "$REPO" +git fetch proposed +git checkout origin/master +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + +echo "Running tests for branch $BRANCH of repo $REPO" +echo "Last commits:" +git log -2 +fi + +python2 /pagure/runtests.py run -f --py2 diff --git a/dev/containers/runtests_py3.sh b/dev/containers/runtests_py3.sh new file mode 100644 index 0000000..b36a14a --- /dev/null +++ b/dev/containers/runtests_py3.sh @@ -0,0 +1,25 @@ +#!/bin/bash + + +ls -l / + +echo "============== ENVIRONMENT =============" +/usr/bin/env +echo "============== END ENVIRONMENT =============" + +if [ -n "$REPO" -a -n "$BRANCH" ]; then +git remote rm proposed || true +git gc --auto +git remote add proposed "$REPO" +git fetch proposed +git checkout origin/master +git config --global user.email "you@example.com" +git config --global user.name "Your Name" +git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + +echo "Running tests for branch $BRANCH of repo $REPO" +echo "Last commits:" +git log -2 +fi + +python3 /pagure/runtests.py run -f --py3 diff --git a/dev/containers/test_env_template b/dev/containers/test_env_template deleted file mode 100644 index dc5869b..0000000 --- a/dev/containers/test_env_template +++ /dev/null @@ -1,15 +0,0 @@ -FROM $base_image - -$epel_pkg -$infra_repo - -RUN $pkg_mgr install -y $pkg_list \ - && $pkg_mgr clean all \ -# for the hooks we need to have /usr/bin/python - && cp /usr/bin/python3 /usr/bin/python \ -# for the alembic test we need /usr/bin/alembic - && cp /usr/bin/alembic-3 /usr/bin/alembic - -WORKDIR /pagure -ENTRYPOINT [ "/pagure/runtests.py", "run" ] -CMD [] diff --git a/dev/run-tests-container.py b/dev/run-tests-container.py old mode 100755 new mode 100644 index 1f44b4c..73fe99a --- a/dev/run-tests-container.py +++ b/dev/run-tests-container.py @@ -1,21 +1,8 @@ -#! /usr/bin/python3 +#! /usr/bin/env python + import argparse import os import subprocess as sp -from string import Template - -TEMPLATE = "dev/docker/test_env_template" - -PKG_LIST = "python3-alembic python3-arrow python3-binaryornot \ - python3-bleach python3-blinker python3-chardet python3-cryptography \ - python3-docutils python3-flask python3-fedora-flask \ - python3-flask-wtf python3-bcrypt python3-jinja2 \ - python3-markdown python3-munch python3-openid-cla \ - python3-openid-teams python3-psutil python3-pygit2 python3-pillow \ - python3-sqlalchemy python3-straight-plugin python3-wtforms \ - python3-nose python3-coverage python3-mock python3-mock \ - python3-eventlet python3-flask-oidc python3-flake8 python3-celery \ - python3-redis python3-trololio python3-beautifulsoup4 python3-black redis vim git" def setup_parser(): @@ -56,69 +43,38 @@ if __name__ == "__main__": args = parser.parse_args() if args.centos is True: - base_image = "centos:7" - pkg_mgr = "yum" - epel_pkg = "RUN yum -y install epel-release" - infra_repo = ( - "ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo" - ) - container_name = "pagure-test-centos" - PKG_LIST += "python34 python34-coverage" + container_names = ["pagure-c7-rpms-py2"] + container_files = ["centos7-rpms-py2"] + elif args.fedora is True: + container_names = ["pagure-f29-rpms-py3"] + container_files = ["f29-rpms-py3"] else: - base_image = "registry.fedoraproject.org/fedora:latest" - pkg_mgr = "dnf" - container_name = "pagure-test-fedora" - epel_pkg = "" - infra_repo = "" + container_names = ["pagure-f29-rpms-py3", "pagure-c7-rpms-py2"] + container_files = ["f29-rpms-py3", "centos7-rpms-py2"] - with open(TEMPLATE, "r") as fp: - t = Template(fp.read()) - with open("dev/docker/test_env", "w") as fp: - fp.write( - t.substitute( - base_image=base_image, - pkg_list=PKG_LIST, - pkg_mgr=pkg_mgr, - epel_pkg=epel_pkg, - infra_repo=infra_repo, + for idx, container_name in enumerate(container_names): + if args.skip_build is not False: + print("------ Building Container Image -----") + sp.call( + [ + "podman", + "build", + "--rm", + "-t", + container_name, + "-f", + "dev/containers/%s" % container_files[idx], + "dev/containers", + ] ) - ) - - if args.skip_build is not False: - print("------ Building Docker Image -----") - sp.run( - [ - "podman", - "build", - "--rm", - "-t", - container_name, - "-f", - "dev/docker/test_env", - "dev/docker", - ] - ) - if args.shell: - print("--------- Shelling in the container --------------") - command = [ - "podman", - "run", - "-it", - "--rm", - "--name", - container_name, - "-v", - "{}:/pagure".format(os.getcwd()), - "--entrypoint=/bin/bash", - container_name, - ] - sp.run(command) - else: + result_path = "{}/results_{}".format(os.getcwd(), container_files[idx]) + if not os.path.exists(result_path): + os.mkdir(result_path) - print("--------- Running Test --------------") - sp.run( - [ + if args.shell: + print("--------- Shelling in the container --------------") + command = [ "podman", "run", "-it", @@ -126,8 +82,34 @@ if __name__ == "__main__": "--name", container_name, "-v", - "{}:/pagure".format(os.getcwd()), + "{}/results_{}:/pagure/results".format( + os.getcwd(), container_files[idx]), + "-e", + "BRANCH=$BRANCH", + "-e", + "REPO=$REPO", + "--entrypoint=/bin/bash", container_name, - args.test_case, ] - ) + sp.call(command) + else: + print("--------- Running Test --------------") + sp.call( + [ + "podman", + "run", + "-it", + "--rm", + "--name", + container_name, + "-v", + "{}/results_{}:/pagure/results".format( + os.getcwd(), container_files[idx]), + "-e", + "BRANCH={}".format(os.environ.get("BRANCH") or ""), + "-e", + "REPO={}".format(os.environ.get("REPO") or ""), + container_name, + args.test_case, + ] + ) diff --git a/run_ci_tests_containers.sh b/run_ci_tests_containers.sh new file mode 100644 index 0000000..38e3d14 --- /dev/null +++ b/run_ci_tests_containers.sh @@ -0,0 +1,38 @@ +set -x + +yum install -y podman + +sysctl -w fs.file-max=2048 + +set -e + +echo $BRANCH $REPO + +podman build --rm -t pagure-f29-rpms-py3 \ + -f dev/containers/f29-rpms-py3 \ + dev/containers + +if [ ! -d `pwd`/results_f29-rpms-py3 ]; then + mkdir `pwd`/results_f29-rpms-py3; +fi + +podman run --rm -it --name pagure-f29-rpms-py3 \ + -v `pwd`/results_f29-rpms-py3:/pagure/results \ + -e BRANCH=$BRANCH \ + -e REPO=$REPO \ + pagure-f29-rpms-py3 + + +podman build --rm -t pagure-c7-rpms-py2 \ + -f dev/containers/centos7-rpms-py2 \ + dev/containers + +if [ ! -d `pwd`/results_centos7-rpms-py2 ]; then + mkdir `pwd`/results_centos7-rpms-py2; +fi + +podman run --rm -it --name pagure-c7-rpms-py2 \ + -v `pwd`/results_centos7-rpms-py2:/pagure/results \ + -e BRANCH=$BRANCH \ + -e REPO=$REPO \ + pagure-c7-rpms-py2 From a53aa8657f84a0452ab167332d9292c50d252c4d Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 07 2019 14:41:35 +0000 Subject: [PATCH 16/18] Fix runtests.py so it works in containers Basically in a container we want to mount the results directory so the test results are exported outside of the container. This means, the results directory cannot be deleted or created from inside the container. So if we can delete it, let's just remove all the files it contains so we start fresh. Signed-off-by: Pierre-Yves Chibon --- diff --git a/runtests.py b/runtests.py index cb55090..74a9e13 100755 --- a/runtests.py +++ b/runtests.py @@ -293,9 +293,17 @@ def do_run(args): ) return 1 else: - shutil.rmtree(args.results) - - os.mkdir(args.results) + try: + shutil.rmtree(args.results) + os.mkdir(args.results) + except: + print( + "Could not delete the %s directory, it will be " + "wiped clean" % args.results) + for content in os.listdir(args.results): + os.remove(content) + else: + os.mkdir(args.results) print("Pre-flight checks passed") From f6c8d860ad917c398518dc73b985356a72c18cf5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 07 2019 14:41:35 +0000 Subject: [PATCH 17/18] Move to container-based testing on jenkins Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 838035a..480a376 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -62,7 +62,7 @@ node('pagure') { stage('Run Test Suite') { timeout(time: 6, unit: 'HOURS') { - onmyduffynode 'cd pagure && sh ./run_ci_tests.sh' + onmyduffynode 'cd pagure && sh ./run_ci_tests_containers.sh' } } From ad955e2c2480747ef1ea9db17fd38e09bd6e9bdb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 07 2019 14:51:40 +0000 Subject: [PATCH 18/18] Save the artifacts in jenkins Signed-off-by: Pierre-Yves Chibon --- diff --git a/.cico.pipeline b/.cico.pipeline index 480a376..11fc913 100644 --- a/.cico.pipeline +++ b/.cico.pipeline @@ -71,8 +71,8 @@ node('pagure') { throw e } finally { stage('Sync Artifacts'){ - syncfromduffynode('pagure/results-py2/') - syncfromduffynode('pagure/results-py3/') + syncfromduffynode('pagure/results_f29-rpms-py3/') + syncfromduffynode('pagure/results_centos7-rpms-py2/') } stage('Deallocate Node'){ @@ -85,8 +85,8 @@ node('pagure') { } stage('Archive Artifacts'){ - archiveArtifacts artifacts: 'pagure/results-py2/' - archiveArtifacts artifacts: 'pagure/results-py3/' + archiveArtifacts artifacts: 'pagure/results_f29-rpms-py3/' + archiveArtifacts artifacts: 'pagure/results_centos7-rpms-py2/' } } }