From da98d7c0f657a43511ac72e0d3c2e7190cdcab3b Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Sep 13 2018 12:32:11 +0000 Subject: Use a container environment to run the CI tests. Let's use containers to run in parallel the tests on fedora and on centos. Signed-off-by: Clement Verna --- diff --git a/.gitignore b/.gitignore index 6011ba8..3e8b0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -47,8 +47,9 @@ Vagrantfile *.retry .vagrant -#Ignore generated dockerfile -dev/docker/test_env +#Ignore copied requirements file +dev/ci/requirements* +dev/ci/tests_* # Tox virtualenvs .tox diff --git a/dev/ci/entrypoint.sh b/dev/ci/entrypoint.sh new file mode 100755 index 0000000..49185a1 --- /dev/null +++ b/dev/ci/entrypoint.sh @@ -0,0 +1,6 @@ +if hash python 2>/dev/null; then + python setup.py install +else + python3 setup.py install +fi +nosetests --with-coverage --cover-erase --cover-package=pagure $* diff --git a/dev/ci/fedora-infra-tags.repo b/dev/ci/fedora-infra-tags.repo new file mode 100644 index 0000000..4bd9b21 --- /dev/null +++ b/dev/ci/fedora-infra-tags.repo @@ -0,0 +1,6 @@ +[infrastructure-tags] +name=Fedora Infrastructure tag epel7-infra - x86_64 +baseurl=https://kojipkgs.fedoraproject.org/repos-dist/epel7-infra/latest/x86_64/ +enabled=1 +gpgcheck=1 +gpgkey=https://infrastructure.fedoraproject.org/repo/infra/RPM-GPG-KEY-INFRA-TAGS diff --git a/dev/ci/pagure-tests-centos b/dev/ci/pagure-tests-centos new file mode 100644 index 0000000..1174d05 --- /dev/null +++ b/dev/ci/pagure-tests-centos @@ -0,0 +1,18 @@ +FROM centos:7 + +RUN yum -y install epel-release +ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo + +RUN yum install -y python-alembic python-arrow python-binaryornot \ + python-bleach python-blinker python-chardet python-cryptography \ + python-docutils python-enum34 python-flask python2-fedora-flask \ + python-flask-wtf python2-bcrypt python-jinja2 python-flask-multistatic \ + python-markdown python-munch python-openid-cla python-openid-teams \ + python-psutil python-pygit2 python2-pillow python-pygments python-enum34 \ + python-sqlalchemy python-straight-plugin python-wtforms python-nose \ + python-coverage python-mock python-mock python-eventlet python2-flask-oidc \ + python-flake8 python-celery python-redis python-trololio python-beautifulsoup4 redis git + +WORKDIR /pagure +ENTRYPOINT ["/usr/bin/bash", "/pagure/dev/ci/entrypoint.sh"] +CMD [] diff --git a/dev/ci/pagure-tests-fedora b/dev/ci/pagure-tests-fedora new file mode 100644 index 0000000..db9091b --- /dev/null +++ b/dev/ci/pagure-tests-fedora @@ -0,0 +1,15 @@ +FROM registry.fedoraproject.org/fedora:latest + +RUN dnf -y install @c-development redhat-rpm-config python3-devel libgit2-devel redis openssh\ + git && dnf clean all +ADD ./requirements.txt /tmp +ADD ./requirements-ci.txt /tmp +ADD ./tests_requirements.txt /tmp +# Make sure that we are using the correct version of pygit2 +RUN sed -i '/pygit2/c\pygit2 <= '"`rpm -q libgit2 --queryformat='%{version}'`" /tmp/requirements.txt +RUN pip3 install -r /tmp/requirements.txt -r /tmp/requirements-ci.txt -r /tmp/tests_requirements.txt + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +WORKDIR /pagure +ENTRYPOINT ["/usr/bin/bash", "/pagure/dev/ci/entrypoint.sh"] +CMD [] diff --git a/dev/docker/fedora-infra-tags.repo b/dev/docker/fedora-infra-tags.repo deleted file mode 100644 index 4bd9b21..0000000 --- a/dev/docker/fedora-infra-tags.repo +++ /dev/null @@ -1,6 +0,0 @@ -[infrastructure-tags] -name=Fedora Infrastructure tag epel7-infra - x86_64 -baseurl=https://kojipkgs.fedoraproject.org/repos-dist/epel7-infra/latest/x86_64/ -enabled=1 -gpgcheck=1 -gpgkey=https://infrastructure.fedoraproject.org/repo/infra/RPM-GPG-KEY-INFRA-TAGS diff --git a/dev/docker/test_env_template b/dev/docker/test_env_template deleted file mode 100644 index e7e6994..0000000 --- a/dev/docker/test_env_template +++ /dev/null @@ -1,12 +0,0 @@ -FROM $base_image - -$epel_pkg -$infra_repo - -RUN $pkg_mgr install -y $pkg_list \ - && $pkg_mgr clean all - -ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 -WORKDIR /code -ENTRYPOINT ["/code/runtests.sh"] -CMD [] \ No newline at end of file diff --git a/dev/run-tests-docker.py b/dev/run-tests-docker.py index f1531ac..55448e6 100755 --- a/dev/run-tests-docker.py +++ b/dev/run-tests-docker.py @@ -2,65 +2,79 @@ import argparse import os import subprocess as sp -from string import Template -TEMPLATE = 'dev/docker/test_env_template' - -PKG_LIST = 'python-alembic python-arrow python-binaryornot \ \n'\ - 'python-bleach python-blinker python-chardet python-cryptography \ \n'\ - 'python-docutils python-enum34 python-flask python2-fedora-flask \ \n'\ - 'python-flask-wtf python2-bcrypt python-jinja2 \ \n'\ - 'python-markdown python-munch python-openid-cla python-openid-teams \ \n'\ - 'python-psutil python-pygit2 python2-pillow \ \n'\ - 'python-sqlalchemy python-straight-plugin python-wtforms python-nose \ \n'\ - 'python-coverage python-mock python-mock python-eventlet python2-flask-oidc \ \n'\ - 'python-flake8 python-celery python-redis python-trololio python-beautifulsoup4 redis vim git' +from shutil import copyfile def setup_parser(): """ Setup the cli arguments """ - parser = argparse.ArgumentParser(prog='pagure-test') - parser.add_argument('test_case', nargs='?', default='', - help='Run the given test case') - parser.add_argument('--fedora', action='store_true', - help='Run the tests in fedora environment (DEFAULT)') - parser.add_argument('--centos', action='store_true', - help='Run the tests in centos environment') - parser.add_argument('--skip-build', dest='skip_build', action='store_false', - help='Skip building the container image') + parser = argparse.ArgumentParser(prog="pagure-test") + parser.add_argument( + "test_case", nargs="?", default="", help="Run the given test case" + ) + parser.add_argument( + "--fedora", + action="store_true", + help="Run the tests in fedora environment (DEFAULT)", + ) + parser.add_argument( + "--centos", + action="store_true", + help="Run the tests in centos environment", + ) + parser.add_argument( + "--skip-build", + dest="skip_build", + action="store_false", + help="Skip building the container image", + ) return parser -if __name__ == '__main__': +if __name__ == "__main__": parser = setup_parser() args = parser.parse_args() if args.centos is True: - base_image = 'centos:7' - pkg_mgr = 'yum' - epel_pkg = 'RUN yum -y install epel-release' - infra_repo = 'ADD ./fedora-infra-tags.repo /etc/yum.repos.d/infra-tags.repo' - container_name = 'pagure-test-centos' + container_name = "pagure-tests-centos" else: - base_image = 'registry.fedoraproject.org/fedora:28' - pkg_mgr = 'dnf' - container_name = 'pagure-test-fedora' - epel_pkg = '' - infra_repo = '' - - with open(TEMPLATE, 'r') as fp: - t = Template(fp.read()) - with open('dev/docker/test_env', 'w') as fp: - fp.write(t.substitute(base_image=base_image, pkg_list=PKG_LIST, - pkg_mgr=pkg_mgr, epel_pkg=epel_pkg, - infra_repo=infra_repo)) + container_name = "pagure-tests-fedora" + copyfile("requirements.txt", "dev/ci/requirements.txt") + copyfile("requirements-ci.txt", "dev/ci/requirements-ci.txt") + copyfile("tests_requirements.txt", "dev/ci/tests_requirements.txt") if args.skip_build is not False: - print('------ Building Docker Image -----') - sp.run(['docker', 'build', '--rm', '-t', container_name, '-f', - 'dev/docker/test_env', 'dev/docker']) + print("------ Building Container Image -----") + sp.run( + [ + "docker", + "build", + "--rm", + "-t", + container_name, + "-f", + "dev/ci/{}".format(container_name), + "dev/ci", + ] + ) - print('--------- Running Test --------------') - sp.run(['docker', 'run', '-it', '--rm', '--name', container_name, '-v', - '{}:/code:z'.format(os.getcwd()), container_name, args.test_case]) + print("--------- Running Test --------------") + sp.run( + [ + "docker", + "run", + "-it", + "--rm", + "--name", + container_name, + "-v", + "{}:/pagure:z".format(os.getcwd()), + "-e", + "PAGURE_CONFIG=/pagure/tests/test_config", + "-e", + "PYTHONPATH=/pagure", + container_name, + args.test_case, + ] + ) diff --git a/run_ci_tests.sh b/run_ci_tests.sh index c55759e..1621e8f 100755 --- a/run_ci_tests.sh +++ b/run_ci_tests.sh @@ -1,15 +1,3 @@ -set -x - -yum install -y python-virtualenv python34 python34-devel \ - gcc python-cryptography python34-cryptography \ - libgit2 libgit2-devel python-pygit2 \ - redis swig openssl-devel m2crypto \ - python2-fedmsg python34-fedmsg-core fedmsg \ - python-tox python-pip python34-pip \ - parallel zeromq-devel python-Cython - -sysctl -w fs.file-max=2048 - set -e echo "============== ENVIRONMENT =============" @@ -31,19 +19,6 @@ echo "Last commits:" git log -2 fi -# Apparently newer requests has strong feeling about idna: -# https://github.com/requests/requests/commit/991e8b76b7a9d21f698b24fa -# and only in py3 we're having a version that is too old (2.1) -pip3 install --upgrade "idna<2.8" -pip install --upgrade tox trollius -pip install --upgrade --force-reinstall chardet -pip3 install "pygit2 <= `rpm -q libgit2 --queryformat='%{version}'`" parallel -v ::: \ -"tox --sitepackages -e 'py27-flask011-ci' -- -v --with-xcoverage --cover-erase --cover-package=pagure" \ -"tox --sitepackages -e 'py34-flask011-ci' -- -v --with-xcoverage --cover-erase --cover-package=pagure" - - -set +e - -tox --sitepackages -e pylint -- -f parseable | tee pylint.out -tox --sitepackages -e lint | tee flake8.out +"./dev/run-tests-docker.py" \ +"./dev/run-tests-docker.py --centos"