From 309f05f4af664870df110324b9d543a31dc0d0cc Mon Sep 17 00:00:00 2001 From: Andrew Engelbrecht Date: Apr 17 2020 18:19:06 +0000 Subject: Upgrade the Docker-based dev environment The containers now all inherit from a common Fedora 31 image, with only Python 3 packages installed. This saves disk space, and creates a consistent testing environment. The base Dockerfile installs Python packages from the distro repository instead of PyPi. This creates a stable installation that isn't dependent on the latest version of any package, so it should work with less adjustment in the future. --- diff --git a/README.rst b/README.rst index aa0c348..63d1c62 100644 --- a/README.rst +++ b/README.rst @@ -47,30 +47,20 @@ Install it `with these instructions `_ For more information about docker-compose cli, see: https://docs.docker.com/compose/reference/. -Once installed, create the folder that will receive the projects, forks, docs, -requests and tickets' git repo. Run this command exactly as it appears from -within the cloned git repo folder:: +To build and run the containers, use the following command:: - $ mkdir -p lcl/{repos,remotes,attachments,releases} - -A docker compose environment is available to run pagure. First use the following -command to build the containers. :: - - $ docker-compose -f dev/docker-compose.yml build - -Once all the containers are built, run the following command to start the containers. :: - - $ docker-compose -f dev/docker-compose.yml up + $ ./dev/docker-start.sh Once all the containers have started, you can access pagure on http://localhost:5000. To stop the containers, press Ctrl+C. -Once the containers are up and running after running the previous command, run -this command to populate the container with test data and create a new account :: +Once the containers are up and running, run this command to populate the +container with test data and create a new account :: - $ docker-compose -f dev/docker-compose.yml exec web python dev-data.py --all + $ docker-compose -f dev/docker-compose.yml exec web python3 dev-data.py --all You can then login with any of the created users, by example: + - username: pingou - password: testing123 diff --git a/dev/containers/base b/dev/containers/base new file mode 100644 index 0000000..5b58ce6 --- /dev/null +++ b/dev/containers/base @@ -0,0 +1,25 @@ +FROM registry.fedoraproject.org/fedora:31 +LABEL maintainers="Patrick Uiterwijk , Andrew Engelbrecht " + +# using distro packages instead of pip, for a stable base image + +RUN dnf -y update && \ + dnf -y install findutils gcc git libgit2-devel python3-alembic \ + python3-arrow python3-bcrypt python3-beautifulsoup4 \ + python3-binaryornot python3-black python3-bleach \ + python3-blinker python3-celery python3-chardet \ + python3-coverage python3-cryptography python3-devel \ + python3-docutils python3-eventlet python3-fedmsg \ + python3-fedora python3-fedora-flask python3-filelock \ + python3-flake8 python3-flask python3-flask-oidc \ + python3-flask-wtf python3-funcsigs python3-jinja2 \ + python3-kitchen python3-markdown python3-mock \ + python3-munch python3-nose python3-nose-xcover \ + python3-openid python3-openid-cla python3-openid-teams \ + python3-pillow python3-psutil python3-psycopg2 \ + python3-pygit2 python3-redis python3-requests \ + python3-setuptools python3-six python3-sqlalchemy \ + python3-straight-plugin python3-trololio \ + python-unversioned-command python3-wtforms which && \ + dnf clean all + diff --git a/dev/containers/ev b/dev/containers/ev index 005ee26..c6065d9 100644 --- a/dev/containers/ev +++ b/dev/containers/ev @@ -1,21 +1,12 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk +FROM pagure-base:latest +LABEL maintainers="Patrick Uiterwijk , Andrew Engelbrecht " VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python2-devel python-setuptools python-nose py-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-markdown python-psutil \ - python-pygit2 python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python-trololio \ - python-celery +RUN mkdir /code WORKDIR /code -ENTRYPOINT ["/usr/bin/python", "/code/pagure-ev/pagure_stream_server.py"] + +ENTRYPOINT ["/usr/bin/python3", "/code/pagure-ev/pagure_stream_server.py"] # Code injection is last to make optimal use of caches VOLUME ["/code"] diff --git a/dev/containers/logcom b/dev/containers/logcom index d46512d..9b93bb5 100644 --- a/dev/containers/logcom +++ b/dev/containers/logcom @@ -1,23 +1,11 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk +FROM pagure-base:latest +LABEL maintainers="Patrick Uiterwijk , Andrew Engelbrecht " VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ - python3-arrow python3-binaryornot python3-bleach python3-blinker \ - python3-chardet python3-cryptography python3-docutils python3-flask \ - python3-flask-wtf python3-markdown python3-psutil \ - python3-pygit2 python3-fedora python3-openid python3-openid-cla \ - python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ - python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ - python3-filelock \ - python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ - python3-kitchen - -RUN dnf install -y python3-celery +RUN mkdir /code WORKDIR /code + ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks_services", "worker", "--loglevel", "info", "-Q", "pagure_logcom"] # Code injection is last to make optimal use of caches diff --git a/dev/containers/web b/dev/containers/web index 78594b0..289097d 100644 --- a/dev/containers/web +++ b/dev/containers/web @@ -1,23 +1,15 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk +FROM pagure-base:latest +LABEL maintainers="Patrick Uiterwijk , Andrew Engelbrecht " VOLUME ["/repos"] RUN mkdir /code +WORKDIR /code -RUN dnf install -y python2-devel python-setuptools python-nose python2-bcrypt python-alembic \ - python-arrow python-binaryornot python-bleach python-blinker \ - python-chardet python-cryptography python-docutils python-flask \ - python-flask-wtf python-markdown python-psutil \ - python-pygit2 python-fedora python-openid python-openid-cla \ - python-openid-teams python-straight-plugin python-wtforms python-munch \ - python-enum34 python-redis python-sqlalchemy systemd gitolite3 python-filelock \ - python-fedora-flask python2-pillow python2-psycopg2 python2-celery \ - findutils COPY web-run /run.sh -WORKDIR /code # Openshift: --no-debug ENTRYPOINT ["/usr/bin/bash", "/run.sh"] + EXPOSE 5000 # Code injection is last to make optimal use of caches diff --git a/dev/containers/web-run b/dev/containers/web-run index 15d8668..221370f 100644 --- a/dev/containers/web-run +++ b/dev/containers/web-run @@ -1,14 +1,15 @@ #!/bin/bash -xe -pip install -r requirements-testing.txt -python setup.py build + +python3 setup.py build + if [ ! -f /attachments/inited ]; then echo "Giving Postgres time to start" sleep 10 touch /attachments/inited - PAGURE_CONFIG=/code/dev/openshift.cfg python createdb.py --initial /code/dev/openshift_alembic.ini + PAGURE_CONFIG=/code/dev/openshift.cfg python3 createdb.py --initial /code/dev/openshift_alembic.ini else - alembic --config /code/dev/openshift_alembic.ini upgrade head + alembic-3 --config /code/dev/openshift_alembic.ini upgrade head fi -exec /usr/bin/python /code/runserver.py --host 0.0.0.0 --config /code/dev/openshift.cfg +exec /usr/bin/python3 /code/runserver.py --host 0.0.0.0 --config /code/dev/openshift.cfg diff --git a/dev/containers/worker b/dev/containers/worker index d54c1f6..994e216 100644 --- a/dev/containers/worker +++ b/dev/containers/worker @@ -1,25 +1,11 @@ -FROM registry.fedoraproject.org/fedora:28 -MAINTAINER Patrick Uiterwijk +FROM pagure-base:latest +LABEL maintainers="Patrick Uiterwijk , Andrew Engelbrecht " VOLUME ["/repos"] -RUN mkdir /code - -RUN dnf install -y python3-devel python3-setuptools python3-nose python3-bcrypt python3-alembic \ - python3-arrow python3-binaryornot python3-bleach python3-blinker \ - python3-chardet python3-cryptography python3-docutils python3-flask \ - python3-flask-wtf python3-markdown python3-psutil \ - python3-pygit2 python3-fedora python3-openid python3-openid-cla \ - python3-openid-teams python3-straight-plugin python3-wtforms python3-munch \ - python3-enum34 python3-redis python3-sqlalchemy systemd gitolite3 \ - python3-filelock python3-bleach python3-cryptography \ - python3-fedora-flask python3-pillow python3-psycopg2 python3-requests \ - python3-blinker - -RUN dnf install -y python3-celery - -RUN ln -sf /usr/bin/python3 /usr/bin/python +RUN mkdir /code WORKDIR /code + ENTRYPOINT ["/usr/bin/celery-3", "-A", "pagure.lib.tasks", "worker", "--loglevel", "info"] # Code injection is last to make optimal use of caches diff --git a/dev/docker-start.sh b/dev/docker-start.sh new file mode 100755 index 0000000..bd4e44c --- /dev/null +++ b/dev/docker-start.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +set -e + +[[ -f dev/docker-compose.yml ]] || (echo "please run this command from the root of the pagure git repo" && exit 1) + +mkdir -p lcl/{repos,remotes,attachments,releases} + +docker image build -f dev/containers/base -t pagure-base . + +docker-compose -f dev/docker-compose.yml build + +docker-compose -f dev/docker-compose.yml up +