From 1b2a4d01f25e92496287645d579fc797fcfd7391 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 10:30:01 +0000 Subject: [PATCH 1/24] Be more compatible --- diff --git a/hubs/migrations/env.py b/hubs/migrations/env.py index 0148bb7..34f4ab0 100644 --- a/hubs/migrations/env.py +++ b/hubs/migrations/env.py @@ -15,6 +15,7 @@ except ImportError: raise sys.path.append(os.getcwd()) +import hubs.app from hubs.models import BASE From 5f12f0e54eb40d54db1d93f0d7435453dd2afd38 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 10:30:17 +0000 Subject: [PATCH 2/24] Ignore local vagrant dir --- diff --git a/.gitignore b/.gitignore index c8921e7..f769f52 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ hubs.js # Vagrant and Ansible files Vagrantfile +.vagrant .dnf-cache *.retry *.pem From 373984121b0475d6c80a5e63342e8dbcf12c19be Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 10:33:25 +0000 Subject: [PATCH 3/24] Paramertize the local user --- diff --git a/ansible/roles/hubs-dev/defaults/main.yml b/ansible/roles/hubs-dev/defaults/main.yml new file mode 100644 index 0000000..746731f --- /dev/null +++ b/ansible/roles/hubs-dev/defaults/main.yml @@ -0,0 +1 @@ +main_user: hubs diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index 9f2ec6b..ef3fb7d 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -62,7 +62,7 @@ # Add various helpful configuration files - name: Install a custom bashrc - copy: src=bashrc dest=/home/{{ ansible_env.SUDO_USER }}/.bashrc + copy: src=bashrc dest=/home/{{ main_user }}/.bashrc - name: Install the message of the day copy: src=motd dest=/etc/motd @@ -111,69 +111,69 @@ # Set up the Python development environment - name: Install Fedora Hubs requirements.txt into hubs virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" pip: - requirements: /home/{{ ansible_env.SUDO_USER }}/devel/requirements.txt - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + requirements: /home/{{ main_user }}/devel/requirements.txt + virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" pip: - requirements: /home/{{ ansible_env.SUDO_USER }}/devel/requirements.txt - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + requirements: /home/{{ main_user }}/devel/requirements.txt + virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ - name: Install bleach into hubs virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" pip: name: bleach - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ - name: Install gunicorn into hubs virtualenv - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" pip: name: gunicorn - virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ - name: Update httplib2 trust store copy: src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem - dest: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt + dest: /home/{{ main_user }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt remote_src: True - name: Add a basic Hubs configuration file - copy: src=hubs_config dest=/home/{{ ansible_env.SUDO_USER }}/devel/config - become_user: "{{ ansible_env.SUDO_USER }}" + copy: src=hubs_config dest=/home/{{ main_user }}/devel/config + become_user: "{{ main_user }}" - name: Configure application to authenticate with iddev.fedorainfracloud.org shell: > source ~/.bashrc && workon hubs && oidc-register https://iddev.fedorainfracloud.org/ http://localhost:5000 - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" args: creates: client_secrets.json - chdir: "/home/{{ ansible_env.SUDO_USER }}/devel/" + chdir: "/home/{{ main_user }}/devel/" - name: Populate the Fedora Hubs database shell: > source ~/.bashrc && workon hubs && python populate.py - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" args: creates: /var/tmp/hubs.db - chdir: "/home/{{ ansible_env.SUDO_USER }}/devel/" + chdir: "/home/{{ main_user }}/devel/" # Set up JavaScript requirements - name: Install npm packages command: npm install - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" args: creates: node_modules - chdir: /home/{{ ansible_env.SUDO_USER }}/devel/hubs/static/client + chdir: /home/{{ main_user }}/devel/hubs/static/client - name: Build JavaScript assests command: node_modules/.bin/webpack - become_user: "{{ ansible_env.SUDO_USER }}" + become_user: "{{ main_user }}" args: - chdir: /home/{{ ansible_env.SUDO_USER }}/devel/hubs/static/client + chdir: /home/{{ main_user }}/devel/hubs/static/client diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index 14cbf9f..b69920c 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -4,4 +4,5 @@ become_method: sudo vars: roles: - - hubs-dev + - role: hubs-dev + main_user: vagrant From 07ddc938861dd557111f0c99a81b624f4c386515 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 10:34:41 +0000 Subject: [PATCH 4/24] Take the setting of motd out of the role --- diff --git a/ansible/files/motd b/ansible/files/motd new file mode 100644 index 0000000..f3d92ab --- /dev/null +++ b/ansible/files/motd @@ -0,0 +1,18 @@ + +Welcome to the Fedora Hubs development environment! + +Here are some tips: + +* The code for Fedora Hubs is located at ~/devel/ + +* You can type `workon hubs` to enter a configured Python virtualenv + +* Run `hup` to start gunicorn + +* Run `hreset` to delete the database and repopulate it + +Once you run `hup` you can navigate to http://localhost:5000/ +in your browser. + +Happy hacking! + diff --git a/ansible/roles/hubs-dev/files/motd b/ansible/roles/hubs-dev/files/motd deleted file mode 100644 index f3d92ab..0000000 --- a/ansible/roles/hubs-dev/files/motd +++ /dev/null @@ -1,18 +0,0 @@ - -Welcome to the Fedora Hubs development environment! - -Here are some tips: - -* The code for Fedora Hubs is located at ~/devel/ - -* You can type `workon hubs` to enter a configured Python virtualenv - -* Run `hup` to start gunicorn - -* Run `hreset` to delete the database and repopulate it - -Once you run `hup` you can navigate to http://localhost:5000/ -in your browser. - -Happy hacking! - diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index ef3fb7d..e5359a0 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -64,9 +64,6 @@ - name: Install a custom bashrc copy: src=bashrc dest=/home/{{ main_user }}/.bashrc -- name: Install the message of the day - copy: src=motd dest=/etc/motd - # Set up Postgres, create the necessary databases, and start up datanommer - name: Set up postgresql database diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index b69920c..d6a4449 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -6,3 +6,7 @@ roles: - role: hubs-dev main_user: vagrant + + tasks: + - name: Install the message of the day + copy: src=files/motd dest=/etc/motd From 5f7827cb014f4be6ce5f6a44534f02d40bc5a8dc Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 14:44:27 +0000 Subject: [PATCH 5/24] PostgreSQL compatibility --- diff --git a/hubs/models.py b/hubs/models.py index d54dccd..0a58e40 100644 --- a/hubs/models.py +++ b/hubs/models.py @@ -120,7 +120,8 @@ class Association(BASE): user_id = sa.Column(sa.Text, sa.ForeignKey('users.username'), primary_key=True) - role = sa.Column(sa.Enum(*roles), primary_key=True) + role = sa.Column( + sa.Enum(*roles, name="roles"), primary_key=True) user = relation("User", backref=backref( 'associations', cascade="all, delete, delete-orphan")) @@ -327,7 +328,8 @@ class HubConfig(BASE): chat_domain = sa.Column(sa.String(256), nullable=True) auth_group = sa.Column(sa.String(256), nullable=True) visibility = sa.Column( - sa.Enum(*VISIBILITY), default="public", nullable=False) + sa.Enum(*VISIBILITY, name="hub_visibility"), + default="public", nullable=False) def __json__(self): return { @@ -362,12 +364,13 @@ class Widget(ObjectAuthzMixin, BASE): plugin = sa.Column(sa.String(50), nullable=False) created_on = sa.Column(sa.DateTime, default=datetime.datetime.utcnow) hub_id = sa.Column(sa.String(50), sa.ForeignKey('hubs.name')) - _config = sa.Column(sa.String(256), default=_widget_config_default) + _config = sa.Column(sa.Text, default=_widget_config_default) index = sa.Column(sa.Integer, nullable=False) left = sa.Column(sa.Boolean, nullable=False, default=False) visibility = sa.Column( - sa.Enum(*VISIBILITY), default="public", nullable=False) + sa.Enum(*VISIBILITY, name="widget_visibility"), + default="public", nullable=False) @classmethod def by_idx(cls, idx): From dfff7b60df14cb6353888b9bb42814946c29f30f Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 16 2017 15:40:25 +0000 Subject: [PATCH 6/24] Create console scripts --- diff --git a/hubs/backend/triage.py b/hubs/backend/triage.py index 7f7858c..50ba5ef 100755 --- a/hubs/backend/triage.py +++ b/hubs/backend/triage.py @@ -73,7 +73,8 @@ def triage(outbound, msg): })) -def main(args): +def main(args=None): + args = args if args is not None else sys.argv logging.config.dictConfig(fedmsg_config['logging']) logging.basicConfig() diff --git a/hubs/backend/worker.py b/hubs/backend/worker.py index 8e8d8c6..30c55bc 100755 --- a/hubs/backend/worker.py +++ b/hubs/backend/worker.py @@ -66,7 +66,8 @@ def handle(idx, fn_name): session.close() -def main(args): +def main(args=None): + args = args if args is not None else sys.argv logging.config.dictConfig(fedmsg_config['logging']) logging.basicConfig() diff --git a/setup.py b/setup.py index 824eeb9..7ca61b7 100644 --- a/setup.py +++ b/setup.py @@ -42,5 +42,9 @@ setup( 'moksha.consumer': [ "cache_invalidator = hubs.backend.consumer:CacheInvalidatorExtraordinaire", # noqa: E501 ], + 'console_scripts': [ + "fedora-hubs-triage = hubs.backend.triage:main", + "fedora-hubs-worker = hubs.backend.worker:main", + ], }, ) diff --git a/systemd/hubs-triage@.service b/systemd/hubs-triage@.service index c88ebde..400ddd7 100644 --- a/systemd/hubs-triage@.service +++ b/systemd/hubs-triage@.service @@ -4,8 +4,9 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/srv/git/fedora-hubs/hubs/backend/triage.py -WorkingDirectory=/srv/git/fedora-hubs/ +ExecStart=/usr/bin/fedora-hubs-triage +WorkingDirectory=/srv/hubs/fedora-hubs/ +Environment=HUBS_CONFIG=/srv/hubs/fedora-hubs/config Type=simple User=root Group=root diff --git a/systemd/hubs-webapp.service b/systemd/hubs-webapp.service index 5c66951..7bee708 100644 --- a/systemd/hubs-webapp.service +++ b/systemd/hubs-webapp.service @@ -5,7 +5,8 @@ Documentation=https://pagure.io/fedora-hubs/ [Service] ExecStart=/usr/bin/gunicorn -b 0.0.0.0:80 --threads 12 --log-config logging.ini --reload hubs.app:app -WorkingDirectory=/srv/git/fedora-hubs/ +WorkingDirectory=/srv/hubs/fedora-hubs/ +Environment=HUBS_CONFIG=/srv/hubs/fedora-hubs/config Type=simple User=root Group=root diff --git a/systemd/hubs-worker@.service b/systemd/hubs-worker@.service index bf6f0ef..a84f3de 100644 --- a/systemd/hubs-worker@.service +++ b/systemd/hubs-worker@.service @@ -4,8 +4,8 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart=/srv/git/fedora-hubs/hubs/backend/worker.py -WorkingDirectory=/srv/git/fedora-hubs/ +ExecStart=/usr/bin/fedora-hubs-worker +Environment=HUBS_CONFIG=/srv/git/fedora-hubs/config Type=simple User=root Group=root From ffb9f27c98ccaf360c841ff4c8174410fc65515e Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 15:42:53 +0000 Subject: [PATCH 7/24] Don't use fedmsg.d for default values Otherwise those defaults can't be overridden by values in /etc/fedmsg.d. --- diff --git a/fedmsg.d/testconfig.py b/fedmsg.d/testconfig.py deleted file mode 100644 index 8f4f105..0000000 --- a/fedmsg.d/testconfig.py +++ /dev/null @@ -1,56 +0,0 @@ -config = { - - 'hubs.consumer.enabled': True, - # Production ready. - 'hubs.sqlalchemy.uri': 'sqlite:////var/tmp/hubs.db', - - # For communication between fedmsg-hub and the cache worker farm - 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', - 'hubs.redis.work-queue-name': 'fedora-hubs-work-queue', - - # Junk junk junk.. that we don't care about. Drop it! - 'hubs.junk_suffixes': [ - 'buildsys.tag', - 'buildsys.untag', - 'buildsys.rpm.sign', - ], - - 'endpoints': { - "fedora-infrastructure": [ - "tcp://hub.fedoraproject.org:9940", - ] - }, - - 'datanommer.sqlalchemy.uri': - 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - - 'fmn.url': 'https://apps.fedoraproject.org/notifications', - - # Some configuration for the rule processors - "fmn.rules.utils.use_pkgdb2": False, - "fmn.rules.utils.pkgdb2_api_url": "http://209.132.184.188/api/", - "fmn.rules.cache": { - "backend": "dogpile.cache.dbm", - "expiration_time": 300, - "arguments": { - "filename": "/var/tmp/fmn-cache.db", - }, - }, - - # Some configuration for the general hubs cache. - "fedora-hubs.cache": { - "backend": "dogpile.cache.dbm", - "expiration_time": -1, # Expire every 1 second, for development - "arguments": { - "filename": "/var/tmp/fedora-hubs-cache.db", - }, - }, - - # Run datanommer locally, just for development. - 'datanommer.enabled': True, - 'datanommer.sqlalchemy.url': - 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - - # Only do one of these so we can try to not run out of memory. - 'moksha.workers_per_consumer': 1, -} diff --git a/hubs/app.py b/hubs/app.py index fa24829..8072e1f 100644 --- a/hubs/app.py +++ b/hubs/app.py @@ -31,8 +31,7 @@ if 'HUBS_CONFIG' in os.environ: app.config.from_envvar('HUBS_CONFIG') fedmsg_config = get_fedmsg_config() -session = hubs.models.init( - fedmsg_config.get('hubs.sqlalchemy.uri', 'sqlite:///')) +session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri']) datanommer.models.init( fedmsg_config.get('datanommer.sqlalchemy.uri', 'sqlite:///')) diff --git a/hubs/default_fedmsg_config.py b/hubs/default_fedmsg_config.py new file mode 100644 index 0000000..81198d6 --- /dev/null +++ b/hubs/default_fedmsg_config.py @@ -0,0 +1,58 @@ +config = { + + # Fedmsg hub consumer + 'hubs.consumer.enabled': True, + + # Main database + 'hubs.sqlalchemy.uri': 'sqlite:////var/tmp/hubs.db', + + # For communication between fedmsg-hub and the cache worker farm + 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', + 'hubs.redis.work-queue-name': 'fedora-hubs-work-queue', + + # Junk junk junk.. that we don't care about. Drop it! + 'hubs.junk_suffixes': [ + 'buildsys.tag', + 'buildsys.untag', + 'buildsys.rpm.sign', + ], + + 'endpoints': { + "fedora-infrastructure": [ + "tcp://hub.fedoraproject.org:9940", + ] + }, + + 'datanommer.sqlalchemy.uri': + 'postgres://postgres:laksjdf@127.0.0.1/datanommer', + + 'fmn.url': 'https://apps.fedoraproject.org/notifications', + + # Some configuration for the rule processors + "fmn.rules.utils.use_pkgdb2": False, + "fmn.rules.utils.pkgdb2_api_url": "http://209.132.184.188/api/", + "fmn.rules.cache": { + "backend": "dogpile.cache.dbm", + "expiration_time": 300, + "arguments": { + "filename": "/var/tmp/fmn-cache.db", + }, + }, + + # Some configuration for the general hubs cache. + "fedora-hubs.cache": { + "backend": "dogpile.cache.dbm", + "expiration_time": -1, # Expire every 1 second, for development + "arguments": { + "filename": "/var/tmp/fedora-hubs-cache.db", + }, + }, + + # Run datanommer locally, just for development. + 'datanommer.enabled': True, + 'datanommer.sqlalchemy.url': + 'postgres://postgres:laksjdf@127.0.0.1/datanommer', + + # Only do one of these so we can try to not run out of memory. + 'moksha.workers_per_consumer': 1, +} diff --git a/hubs/utils/__init__.py b/hubs/utils/__init__.py index 8713806..80d239f 100755 --- a/hubs/utils/__init__.py +++ b/hubs/utils/__init__.py @@ -7,6 +7,8 @@ import fedmsg.config import fedmsg.meta from six.moves.urllib_parse import urlencode +from hubs.default_fedmsg_config import config as default_fedmsg_config + def get_fedmsg_config(): try: @@ -14,6 +16,11 @@ def get_fedmsg_config(): except KeyError: filenames = None fedmsg_config = fedmsg.config.load_config(filenames=filenames) + # Only add default values if they don't exist yet. + for key, value in default_fedmsg_config.items(): + if key not in fedmsg_config: + fedmsg_config[key] = value + # Meta processors fedmsg.meta.make_processors(**fedmsg_config) return fedmsg_config From d693c057d973a702e6f62c1622d59f202ecb5bbd Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 16:05:55 +0000 Subject: [PATCH 8/24] Use variables in playbooks --- diff --git a/ansible/roles/hubs-dev/defaults/main.yml b/ansible/roles/hubs-dev/defaults/main.yml index 746731f..8f50281 100644 --- a/ansible/roles/hubs-dev/defaults/main.yml +++ b/ansible/roles/hubs-dev/defaults/main.yml @@ -1 +1,8 @@ main_user: hubs +hubs_dev_mode: yes +hubs_secret_key: changeme +hubs_code_dir: "/home/{{ main_user }}/devel" +hubs_venv_dir: "/home/{{ main_user }}/.virtualenvs/hubs" +hubs_db_type: sqlite +hubs_db_password: changeme +hubs_url: http://localhost:5000 diff --git a/ansible/roles/hubs-dev/files/bashrc b/ansible/roles/hubs-dev/files/bashrc deleted file mode 100644 index 615b5df..0000000 --- a/ansible/roles/hubs-dev/files/bashrc +++ /dev/null @@ -1,39 +0,0 @@ -# .bashrc - -# Source global definitions -if [ -f /etc/bashrc ]; then - . /etc/bashrc -fi - -# Uncomment the following line if you don't like systemctl's auto-paging feature: -# export SYSTEMD_PAGER= - -# User specific aliases and functions -# If adding new functions to this file, note that you can add help text to the function -# by defining a variable with name __help containing the help text - -# Set up virtualenvwrapper -export WORKON_HOME=$HOME/.virtualenvs -export PIP_VIRTUALENV_BASE=$WORKON_HOME -export VIRTUALENV_USE_DISTRIBUTE=true -export PIP_RESPECT_VIRTUALENV=true -source /usr/bin/virtualenvwrapper.sh - -export HUBS_CONFIG=~/devel/config - - -hup() { - workon hubs - pushd ~/devel/ - python ./runserver.py --host 0.0.0.0 -} - -hreset() { - workon hubs - rm /var/tmp/hubs.db - rm /var/tmp/fedora-hubs-cache.db - pushd ~/devel/ - python populate.py - popd - deactivate -} diff --git a/ansible/roles/hubs-dev/files/hubs_config b/ansible/roles/hubs-dev/files/hubs_config deleted file mode 100644 index 83417df..0000000 --- a/ansible/roles/hubs-dev/files/hubs_config +++ /dev/null @@ -1,4 +0,0 @@ -# Enter any hubs configuratio here - -# Allow the cookie to be sent of http since we work on localhost -OIDC_ID_TOKEN_COOKIE_SECURE = False diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index e5359a0..b1ae3fc 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -62,7 +62,7 @@ # Add various helpful configuration files - name: Install a custom bashrc - copy: src=bashrc dest=/home/{{ main_user }}/.bashrc + template: src=bashrc dest=/home/{{ main_user }}/.bashrc # Set up Postgres, create the necessary databases, and start up datanommer @@ -110,26 +110,23 @@ - name: Install Fedora Hubs requirements.txt into hubs virtualenv become_user: "{{ main_user }}" pip: - requirements: /home/{{ main_user }}/devel/requirements.txt - virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ + requirements: "{{ hubs_code_dir }}/requirements.txt" + virtualenv: "{{ hubs_venv_dir}}" - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv become_user: "{{ main_user }}" pip: - requirements: /home/{{ main_user }}/devel/requirements.txt - virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ + requirements: "{{ hubs_code_dir }}/test-requirements.txt" + virtualenv: "{{ hubs_venv_dir}}" -- name: Install bleach into hubs virtualenv +- name: Install other packages into hubs virtualenv become_user: "{{ main_user }}" pip: - name: bleach - virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ - -- name: Install gunicorn into hubs virtualenv - become_user: "{{ main_user }}" - pip: - name: gunicorn - virtualenv: /home/{{ main_user }}/.virtualenvs/hubs/ + name: "{{ item }}" + virtualenv: "{{ hubs_venv_dir }}" + with_items: + - bleach + - gunicorn - name: Update httplib2 trust store copy: @@ -138,18 +135,17 @@ remote_src: True - name: Add a basic Hubs configuration file - copy: src=hubs_config dest=/home/{{ main_user }}/devel/config + template: + src: [hubs_config, hubs_config.{{ ansible_group }}, hubs_config.{{ ansible_hostname }}] + dest: "{{ hubs_code_dir }}/config" become_user: "{{ main_user }}" - name: Configure application to authenticate with iddev.fedorainfracloud.org - shell: > - source ~/.bashrc && - workon hubs && - oidc-register https://iddev.fedorainfracloud.org/ http://localhost:5000 + command: oidc-register https://iddev.fedorainfracloud.org/ {{hubs_url}} become_user: "{{ main_user }}" args: creates: client_secrets.json - chdir: "/home/{{ main_user }}/devel/" + chdir: "{{ hubs_code_dir }}" - name: Populate the Fedora Hubs database shell: > @@ -159,7 +155,7 @@ become_user: "{{ main_user }}" args: creates: /var/tmp/hubs.db - chdir: "/home/{{ main_user }}/devel/" + chdir: "{{ hubs_code_dir }}" # Set up JavaScript requirements - name: Install npm packages @@ -167,10 +163,10 @@ become_user: "{{ main_user }}" args: creates: node_modules - chdir: /home/{{ main_user }}/devel/hubs/static/client + chdir: "{{ hubs_code_dir }}/hubs/static/client" - name: Build JavaScript assests command: node_modules/.bin/webpack become_user: "{{ main_user }}" args: - chdir: /home/{{ main_user }}/devel/hubs/static/client + chdir: "{{ hubs_code_dir }}/hubs/static/client" diff --git a/ansible/roles/hubs-dev/templates/bashrc b/ansible/roles/hubs-dev/templates/bashrc new file mode 100644 index 0000000..db0746d --- /dev/null +++ b/ansible/roles/hubs-dev/templates/bashrc @@ -0,0 +1,45 @@ +# .bashrc + +# Source global definitions +if [ -f /etc/bashrc ]; then + . /etc/bashrc +fi + +# Uncomment the following line if you don't like systemctl's auto-paging feature: +# export SYSTEMD_PAGER= + +# User specific aliases and functions +# If adding new functions to this file, note that you can add help text to the function +# by defining a variable with name __help containing the help text + +# Set up virtualenvwrapper +export WORKON_HOME=$HOME/.virtualenvs +export PIP_VIRTUALENV_BASE=$WORKON_HOME +export VIRTUALENV_USE_DISTRIBUTE=true +export PIP_RESPECT_VIRTUALENV=true +source /usr/bin/virtualenvwrapper.sh + +export HUBS_CONFIG={{ hubs_code_dir }}/config +export FLASK_APP={{ hubs_code_dir }}/hubs/app.py + + +hup() { + workon hubs + pushd {{ hubs_code_dir }} + FLASK_DEBUG=1 flask run --host 0.0.0.0 +} + +hreset() { + workon hubs + {% if hubs_db_type == "postgresql" %} + sudo -u postgres dropdb hubs + sudo -u postgres createdb -O hubs hubs + {% else %} + rm /var/lib/hubs/hubs.db + {% endif %} + rm /var/lib/hubs/cache.db + pushd {{ hubs_code_dir }} + python populate.py + popd + deactivate +} diff --git a/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service b/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service new file mode 100644 index 0000000..cc2b96e --- /dev/null +++ b/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service @@ -0,0 +1,15 @@ +[Unit] +Description=Hubs-specific fedmsg processing hub +After=network.target +Documentation=https://fedmsg.readthedocs.org/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub +WorkingDirectory={{ hubs_code_dir }} +Type=simple +User=fedmsg +Group=fedmsg +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-triage@.service b/ansible/roles/hubs-dev/templates/hubs-triage@.service new file mode 100644 index 0000000..fc74242 --- /dev/null +++ b/ansible/roles/hubs-dev/templates/hubs-triage@.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs triage worker #%i +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-webapp.service b/ansible/roles/hubs-dev/templates/hubs-webapp.service new file mode 100644 index 0000000..00ba431 --- /dev/null +++ b/ansible/roles/hubs-dev/templates/hubs-webapp.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs frontend webapp +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:80 --threads 12 --log-config logging.ini --reload hubs.app:app +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-worker@.service b/ansible/roles/hubs-dev/templates/hubs-worker@.service new file mode 100644 index 0000000..8073577 --- /dev/null +++ b/ansible/roles/hubs-dev/templates/hubs-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs cache worker #%i +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs_config b/ansible/roles/hubs-dev/templates/hubs_config new file mode 100644 index 0000000..d5a5394 --- /dev/null +++ b/ansible/roles/hubs-dev/templates/hubs_config @@ -0,0 +1,8 @@ +# Enter any hubs configuration here + +SECRET_KEY = "{{ hubs_secret_key }}" + +{% if hubs_dev_mode %} +# Allow the cookie to be sent of http since we work on localhost +OIDC_ID_TOKEN_COOKIE_SECURE = False +{% endif %} diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index d6a4449..f971611 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -3,9 +3,13 @@ become: true become_method: sudo vars: + main_user: vagrant + roles: - role: hubs-dev - main_user: vagrant + hubs_secret_key: vagrantmachinesecretkey + hubs_db_type: sqlite + hubs_url: http://localhost:5000 tasks: - name: Install the message of the day From e99e1ea318d56d9b9fca0f7b5124deed16add6ac Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 16:35:47 +0000 Subject: [PATCH 9/24] Get rid of datanommer --- diff --git a/ansible/roles/hubs-dev/files/datanommer.py b/ansible/roles/hubs-dev/files/datanommer.py deleted file mode 100644 index 30de6fb..0000000 --- a/ansible/roles/hubs-dev/files/datanommer.py +++ /dev/null @@ -1,4 +0,0 @@ -config = { - 'datanommer.enabled': True, - 'datanommer.sqlalchemy.url': 'postgres://datanommer@localhost/datanommer', -} diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index b1ae3fc..9ec1f23 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -12,8 +12,6 @@ - gcc-c++ - libffi-devel - openssl-devel - - postgresql - - postgresql-devel - python-sphinx - python-virtualenvwrapper - python2-devel @@ -26,7 +24,6 @@ with_items: - python-arrow - python-bleach - - python-datanommer-models - python-decorator - python-dogpile-cache - python-fedmsg-core @@ -38,7 +35,6 @@ - python-gunicorn - python-html5lib - python-munch - - python-psycopg2 - pytz - python-sqlalchemy - python-markdown @@ -48,60 +44,12 @@ - python-pygments-markdown-lexer - python-retask -- name: Install packages for datanommer - dnf: name={{ item }} state=present - with_items: - - datanommer-commands - - fedmsg-hub - - npm - - postgresql-server - - python-datanommer-consumer - - python-fedmsg-meta-fedora-infrastructure - - python-psycopg2 - # Add various helpful configuration files - name: Install a custom bashrc template: src=bashrc dest=/home/{{ main_user }}/.bashrc -# Set up Postgres, create the necessary databases, and start up datanommer -- name: Set up postgresql database - command: postgresql-setup --initdb - args: - creates: /var/lib/pgsql/data/base - -- name: Set up postgresql access rules to allow local access - copy: - src: pg_hba.conf - dest: /var/lib/pgsql/data/pg_hba.conf - owner: postgres - group: postgres - mode: 0600 - notify: restart postgresql - -- name: Start and enable postgresql - service: name=postgresql state=started enabled=yes - -- name: Set up datanommer DB user - postgresql_user: - name: datanommer - role_attr_flags: SUPERUSER,LOGIN - -- name: Create datanommer database - postgresql_db: - name: datanommer - owner: datanommer - -- name: Set up datanommer - copy: src=datanommer.py dest=/etc/fedmsg.d/datanommer.py - notify: restart fedmsg-hub - -# TODO unfortunately this always runs as there doesn't appear to be -# an easy way to check if it needs to run -- name: Create datanommer database tables - command: datanommer-create-db - - name: Start and enable fedmsg-hub service: name=fedmsg-hub state=started enabled=yes diff --git a/hubs/app.py b/hubs/app.py index 8072e1f..1cc6c68 100644 --- a/hubs/app.py +++ b/hubs/app.py @@ -4,7 +4,6 @@ import datetime import logging import os -import datanommer.models import flask import flask.json import munch @@ -32,8 +31,6 @@ if 'HUBS_CONFIG' in os.environ: fedmsg_config = get_fedmsg_config() session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri']) -datanommer.models.init( - fedmsg_config.get('datanommer.sqlalchemy.uri', 'sqlite:///')) @app.before_request diff --git a/hubs/default_fedmsg_config.py b/hubs/default_fedmsg_config.py index 81198d6..648f504 100644 --- a/hubs/default_fedmsg_config.py +++ b/hubs/default_fedmsg_config.py @@ -23,9 +23,6 @@ config = { ] }, - 'datanommer.sqlalchemy.uri': - 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - 'fmn.url': 'https://apps.fedoraproject.org/notifications', # Some configuration for the rule processors @@ -48,11 +45,6 @@ config = { }, }, - # Run datanommer locally, just for development. - 'datanommer.enabled': True, - 'datanommer.sqlalchemy.url': - 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - # Only do one of these so we can try to not run out of memory. 'moksha.workers_per_consumer': 1, } diff --git a/hubs/tests/fedmsg_test.cfg b/hubs/tests/fedmsg_test.cfg index 6d64bdf..6dc85b7 100644 --- a/hubs/tests/fedmsg_test.cfg +++ b/hubs/tests/fedmsg_test.cfg @@ -21,9 +21,6 @@ config = { ] }, - #'datanommer.sqlalchemy.uri': - # 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - #'fmn.url': 'https://apps.fedoraproject.org/notifications', ## Some configuration for the rule processors @@ -42,11 +39,6 @@ config = { "backend": "dogpile.cache.null", }, - ## Run datanommer locally, just for development. - #'datanommer.enabled': True, - #'datanommer.sqlalchemy.url': - # 'postgres://postgres:laksjdf@127.0.0.1/datanommer', - ## Only do one of these so we can try to not run out of memory. #'moksha.workers_per_consumer': 1, } diff --git a/requirements.txt b/requirements.txt index 4a76840..1eb3902 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ arrow bleach<2.0.0 blinker -datanommer.models decorator dogpile.cache enum34 From 2c6b4b74b42f70ee9725dc903db7fb5e34fbf167 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 22:40:54 +0000 Subject: [PATCH 10/24] Deploy a specific configuration file --- diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index 9ec1f23..2912be9 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -88,6 +88,21 @@ dest: "{{ hubs_code_dir }}/config" become_user: "{{ main_user }}" +- name: Add a basic fedmsg configuration file + template: + src: [fedmsg_config, fedmsg_config.{{ ansible_group }}, fedmsg_config.{{ ansible_hostname }}] + dest: "/etc/fedmsg.d/hubs_config.py" + become_user: "{{ main_user }}" + +- name: Create a directory for variable files + file: + path: "/var/lib/hubs" + state: directory + owner: "{{ main_user }}" + group: "{{ main_user }}" + mode: 755 + setype: httpd_sys_content_rw_t + - name: Configure application to authenticate with iddev.fedorainfracloud.org command: oidc-register https://iddev.fedorainfracloud.org/ {{hubs_url}} become_user: "{{ main_user }}" diff --git a/ansible/roles/hubs-dev/templates/fedmsg_config b/ansible/roles/hubs-dev/templates/fedmsg_config new file mode 100644 index 0000000..0b33780 --- /dev/null +++ b/ansible/roles/hubs-dev/templates/fedmsg_config @@ -0,0 +1,15 @@ +config = { + + # Database + 'hubs.sqlalchemy.uri': 'sqlite:////var/lib/hubs/hubs.db', + + # Some configuration for the general hubs cache. + "fedora-hubs.cache": { + "backend": "dogpile.cache.dbm", + #"expiration_time": 0, + "arguments": { + "filename": "/var/lib/hubs/cache.db", + }, + }, +} + diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index f971611..623d093 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -5,6 +5,12 @@ vars: main_user: vagrant + pre_tasks: + - name: Allow the vagrant user to sudo as anyone + copy: + content: "vagrant ALL=(ALL) NOPASSWD: ALL" + dest: /etc/sudoers.d/vagrant-nopasswd + roles: - role: hubs-dev hubs_secret_key: vagrantmachinesecretkey From 3d90f0304ea28913e28817312ad0be188a09e38e Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 22:43:24 +0000 Subject: [PATCH 11/24] Options to deploy using sqlite or postgresql --- diff --git a/ansible/roles/hubs-dev/files/pg_hba.conf b/ansible/roles/hubs-dev/files/pg_hba.conf index fb47623..c703fd9 100644 --- a/ansible/roles/hubs-dev/files/pg_hba.conf +++ b/ansible/roles/hubs-dev/files/pg_hba.conf @@ -7,8 +7,8 @@ # TYPE DATABASE USER ADDRESS METHOD # "local" is for Unix domain socket connections only -local all all trust +local all all peer # IPv4 local connections: -host all all 127.0.0.1/32 trust +host all all 127.0.0.1/32 md5 # IPv6 local connections: -host all all ::1/128 trust +host all all ::1/128 md5 diff --git a/ansible/roles/hubs-dev/tasks/db-postgresql.yml b/ansible/roles/hubs-dev/tasks/db-postgresql.yml new file mode 100644 index 0000000..d228b80 --- /dev/null +++ b/ansible/roles/hubs-dev/tasks/db-postgresql.yml @@ -0,0 +1,54 @@ +# Set up Postgres, create the database, and populate it. + +- name: Install dependencies + dnf: name={{ item }} state=present + with_items: + - postgresql-server + - python-psycopg2 + +- name: Set up postgresql database + command: postgresql-setup --initdb + args: + creates: /var/lib/pgsql/data/base + +- name: Set up postgresql access rules to allow local access + copy: + src: pg_hba.conf + dest: /var/lib/pgsql/data/pg_hba.conf + owner: postgres + group: postgres + mode: 0600 + notify: restart postgresql + +- name: Start and enable postgresql + service: name=postgresql state=started enabled=yes + +- name: Set up the DB user + postgresql_user: + name: hubs + password: "{{ hubs_db_password }}" + role_attr_flags: NOSUPERUSER,NOCREATEROLE,NOCREATEDB + become_user: postgres + +- name: Create the database + postgresql_db: + name: hubs + owner: hubs + register: db_creation + become_user: postgres + +- name: Ease local access to the database + copy: + content: "*:*:hubs:hubs:{{ hubs_db_password }}" + dest: /home/{{ main_user }}/.pgpass + mode: 600 + owner: "{{ main_user }}" + group: "{{ main_user }}" + +- name: Populate the Fedora Hubs database + command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + args: + chdir: "{{ hubs_code_dir }}" + become_user: "{{ main_user }}" + when: db_creation|succeeded + diff --git a/ansible/roles/hubs-dev/tasks/db-sqlite.yml b/ansible/roles/hubs-dev/tasks/db-sqlite.yml new file mode 100644 index 0000000..120e1c7 --- /dev/null +++ b/ansible/roles/hubs-dev/tasks/db-sqlite.yml @@ -0,0 +1,6 @@ +- name: Create and populate the Fedora Hubs database + command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + become_user: "{{ main_user }}" + args: + creates: /var/lib/hubs/hubs.db + chdir: "{{ hubs_code_dir }}" diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index 2912be9..5b3a341 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -110,15 +110,10 @@ creates: client_secrets.json chdir: "{{ hubs_code_dir }}" -- name: Populate the Fedora Hubs database - shell: > - source ~/.bashrc && - workon hubs && - python populate.py - become_user: "{{ main_user }}" - args: - creates: /var/tmp/hubs.db - chdir: "{{ hubs_code_dir }}" + +# Set up, create, and populate the database. +- include: db-{{ hubs_database_type }}.yml + # Set up JavaScript requirements - name: Install npm packages diff --git a/ansible/roles/hubs-dev/templates/fedmsg_config b/ansible/roles/hubs-dev/templates/fedmsg_config index 0b33780..c54fdb2 100644 --- a/ansible/roles/hubs-dev/templates/fedmsg_config +++ b/ansible/roles/hubs-dev/templates/fedmsg_config @@ -1,7 +1,11 @@ config = { # Database + {% if hubs_db_type == "postgresql" %} + 'hubs.sqlalchemy.uri': 'postgresql://hubs:{{ hubs_db_password }}@localhost/hubs', + {% else %} 'hubs.sqlalchemy.uri': 'sqlite:////var/lib/hubs/hubs.db', + {% endif %} # Some configuration for the general hubs cache. "fedora-hubs.cache": { From 66f6dd2a443cdb5704814ddd8804a8679d75afcf Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 22:44:58 +0000 Subject: [PATCH 12/24] Deploy in a virtualenv with access to system packages --- diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index 5b3a341..b55a81a 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -60,18 +60,21 @@ pip: requirements: "{{ hubs_code_dir }}/requirements.txt" virtualenv: "{{ hubs_venv_dir}}" + virtualenv_site_packages: yes - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv become_user: "{{ main_user }}" pip: requirements: "{{ hubs_code_dir }}/test-requirements.txt" virtualenv: "{{ hubs_venv_dir}}" + virtualenv_site_packages: yes - name: Install other packages into hubs virtualenv become_user: "{{ main_user }}" pip: name: "{{ item }}" virtualenv: "{{ hubs_venv_dir }}" + virtualenv_site_packages: yes with_items: - bleach - gunicorn From e41282c3e275b7687ef58bd0a3eec00793f93d31 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 19 2017 22:45:43 +0000 Subject: [PATCH 13/24] Deploy services --- diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml index b55a81a..75c0736 100755 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -5,6 +5,13 @@ - git - vim-enhanced +- name: Install external dependencies + dnf: name={{ item }} state=present + with_items: + - npm + - redis + - fedmsg-hub + - name: Install Fedora Hubs development packages dnf: name={{ item }} state=present with_items: @@ -50,10 +57,6 @@ template: src=bashrc dest=/home/{{ main_user }}/.bashrc -- name: Start and enable fedmsg-hub - service: name=fedmsg-hub state=started enabled=yes - - # Set up the Python development environment - name: Install Fedora Hubs requirements.txt into hubs virtualenv become_user: "{{ main_user }}" @@ -131,3 +134,28 @@ become_user: "{{ main_user }}" args: chdir: "{{ hubs_code_dir }}/hubs/static/client" + +# Services +- name: Disable the system-wide fedmsg-hub + service: name=fedmsg-hub state=stopped enabled=no + +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-webapp + - hubs-triage@ + - hubs-worker@ + - hubs-fedmsg-hub + +- name: Start and enable the services + service: name={{ item }} state=started enabled=yes + with_items: + - redis + - hubs-webapp + - hubs-triage@1 + - hubs-triage@2 + - hubs-worker@1 + - hubs-worker@2 + - hubs-fedmsg-hub From dc681906692edb90839077a8272836967125a00b Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 20 2017 03:42:15 +0000 Subject: [PATCH 14/24] Rename ansible role to be more generic --- diff --git a/ansible/roles/hubs-dev/defaults/main.yml b/ansible/roles/hubs-dev/defaults/main.yml deleted file mode 100644 index 8f50281..0000000 --- a/ansible/roles/hubs-dev/defaults/main.yml +++ /dev/null @@ -1,8 +0,0 @@ -main_user: hubs -hubs_dev_mode: yes -hubs_secret_key: changeme -hubs_code_dir: "/home/{{ main_user }}/devel" -hubs_venv_dir: "/home/{{ main_user }}/.virtualenvs/hubs" -hubs_db_type: sqlite -hubs_db_password: changeme -hubs_url: http://localhost:5000 diff --git a/ansible/roles/hubs-dev/files/pg_hba.conf b/ansible/roles/hubs-dev/files/pg_hba.conf deleted file mode 100644 index c703fd9..0000000 --- a/ansible/roles/hubs-dev/files/pg_hba.conf +++ /dev/null @@ -1,14 +0,0 @@ -# PostgreSQL Client Authentication Configuration File -# =================================================== -# -# Refer to the "Client Authentication" section in the PostgreSQL -# documentation for a complete description of this file. - -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all peer -# IPv4 local connections: -host all all 127.0.0.1/32 md5 -# IPv6 local connections: -host all all ::1/128 md5 diff --git a/ansible/roles/hubs-dev/handlers/main.yml b/ansible/roles/hubs-dev/handlers/main.yml deleted file mode 100644 index 5d4ca3e..0000000 --- a/ansible/roles/hubs-dev/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: restart postgresql - service: name=postgresql state=restarted - -- name: restart fedmsg-hub - service: name=fedmsg-hub state=restarted diff --git a/ansible/roles/hubs-dev/tasks/db-postgresql.yml b/ansible/roles/hubs-dev/tasks/db-postgresql.yml deleted file mode 100644 index d228b80..0000000 --- a/ansible/roles/hubs-dev/tasks/db-postgresql.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Set up Postgres, create the database, and populate it. - -- name: Install dependencies - dnf: name={{ item }} state=present - with_items: - - postgresql-server - - python-psycopg2 - -- name: Set up postgresql database - command: postgresql-setup --initdb - args: - creates: /var/lib/pgsql/data/base - -- name: Set up postgresql access rules to allow local access - copy: - src: pg_hba.conf - dest: /var/lib/pgsql/data/pg_hba.conf - owner: postgres - group: postgres - mode: 0600 - notify: restart postgresql - -- name: Start and enable postgresql - service: name=postgresql state=started enabled=yes - -- name: Set up the DB user - postgresql_user: - name: hubs - password: "{{ hubs_db_password }}" - role_attr_flags: NOSUPERUSER,NOCREATEROLE,NOCREATEDB - become_user: postgres - -- name: Create the database - postgresql_db: - name: hubs - owner: hubs - register: db_creation - become_user: postgres - -- name: Ease local access to the database - copy: - content: "*:*:hubs:hubs:{{ hubs_db_password }}" - dest: /home/{{ main_user }}/.pgpass - mode: 600 - owner: "{{ main_user }}" - group: "{{ main_user }}" - -- name: Populate the Fedora Hubs database - command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" - args: - chdir: "{{ hubs_code_dir }}" - become_user: "{{ main_user }}" - when: db_creation|succeeded - diff --git a/ansible/roles/hubs-dev/tasks/db-sqlite.yml b/ansible/roles/hubs-dev/tasks/db-sqlite.yml deleted file mode 100644 index 120e1c7..0000000 --- a/ansible/roles/hubs-dev/tasks/db-sqlite.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: Create and populate the Fedora Hubs database - command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" - become_user: "{{ main_user }}" - args: - creates: /var/lib/hubs/hubs.db - chdir: "{{ hubs_code_dir }}" diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml deleted file mode 100755 index 75c0736..0000000 --- a/ansible/roles/hubs-dev/tasks/main.yml +++ /dev/null @@ -1,161 +0,0 @@ ---- -- name: Install helpful development packages - dnf: name={{ item }} state=present - with_items: - - git - - vim-enhanced - -- name: Install external dependencies - dnf: name={{ item }} state=present - with_items: - - npm - - redis - - fedmsg-hub - -- name: Install Fedora Hubs development packages - dnf: name={{ item }} state=present - with_items: - - gcc - - gcc-c++ - - libffi-devel - - openssl-devel - - python-sphinx - - python-virtualenvwrapper - - python2-devel - - python3-devel - - redhat-rpm-config - - sqlite-devel - -- name: Install the distribution versions of requirements.txt - dnf: name={{ item }} state=present - with_items: - - python-arrow - - python-bleach - - python-decorator - - python-dogpile-cache - - python-fedmsg-core - - python-fedmsg-meta-fedora-infrastructure - - python-flask - - python3-flask-oidc - - python-fmn-lib - - python-fmn-rules - - python-gunicorn - - python-html5lib - - python-munch - - pytz - - python-sqlalchemy - - python-markdown - - python2-pkgwat-api - - python-six - - python-pygments - - python-pygments-markdown-lexer - - python-retask - - -# Add various helpful configuration files -- name: Install a custom bashrc - template: src=bashrc dest=/home/{{ main_user }}/.bashrc - - -# Set up the Python development environment -- name: Install Fedora Hubs requirements.txt into hubs virtualenv - become_user: "{{ main_user }}" - pip: - requirements: "{{ hubs_code_dir }}/requirements.txt" - virtualenv: "{{ hubs_venv_dir}}" - virtualenv_site_packages: yes - -- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv - become_user: "{{ main_user }}" - pip: - requirements: "{{ hubs_code_dir }}/test-requirements.txt" - virtualenv: "{{ hubs_venv_dir}}" - virtualenv_site_packages: yes - -- name: Install other packages into hubs virtualenv - become_user: "{{ main_user }}" - pip: - name: "{{ item }}" - virtualenv: "{{ hubs_venv_dir }}" - virtualenv_site_packages: yes - with_items: - - bleach - - gunicorn - -- name: Update httplib2 trust store - copy: - src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem - dest: /home/{{ main_user }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt - remote_src: True - -- name: Add a basic Hubs configuration file - template: - src: [hubs_config, hubs_config.{{ ansible_group }}, hubs_config.{{ ansible_hostname }}] - dest: "{{ hubs_code_dir }}/config" - become_user: "{{ main_user }}" - -- name: Add a basic fedmsg configuration file - template: - src: [fedmsg_config, fedmsg_config.{{ ansible_group }}, fedmsg_config.{{ ansible_hostname }}] - dest: "/etc/fedmsg.d/hubs_config.py" - become_user: "{{ main_user }}" - -- name: Create a directory for variable files - file: - path: "/var/lib/hubs" - state: directory - owner: "{{ main_user }}" - group: "{{ main_user }}" - mode: 755 - setype: httpd_sys_content_rw_t - -- name: Configure application to authenticate with iddev.fedorainfracloud.org - command: oidc-register https://iddev.fedorainfracloud.org/ {{hubs_url}} - become_user: "{{ main_user }}" - args: - creates: client_secrets.json - chdir: "{{ hubs_code_dir }}" - - -# Set up, create, and populate the database. -- include: db-{{ hubs_database_type }}.yml - - -# Set up JavaScript requirements -- name: Install npm packages - command: npm install - become_user: "{{ main_user }}" - args: - creates: node_modules - chdir: "{{ hubs_code_dir }}/hubs/static/client" - -- name: Build JavaScript assests - command: node_modules/.bin/webpack - become_user: "{{ main_user }}" - args: - chdir: "{{ hubs_code_dir }}/hubs/static/client" - -# Services -- name: Disable the system-wide fedmsg-hub - service: name=fedmsg-hub state=stopped enabled=no - -- name: Install the service files - template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service - with_items: - - hubs-webapp - - hubs-triage@ - - hubs-worker@ - - hubs-fedmsg-hub - -- name: Start and enable the services - service: name={{ item }} state=started enabled=yes - with_items: - - redis - - hubs-webapp - - hubs-triage@1 - - hubs-triage@2 - - hubs-worker@1 - - hubs-worker@2 - - hubs-fedmsg-hub diff --git a/ansible/roles/hubs-dev/templates/bashrc b/ansible/roles/hubs-dev/templates/bashrc deleted file mode 100644 index db0746d..0000000 --- a/ansible/roles/hubs-dev/templates/bashrc +++ /dev/null @@ -1,45 +0,0 @@ -# .bashrc - -# Source global definitions -if [ -f /etc/bashrc ]; then - . /etc/bashrc -fi - -# Uncomment the following line if you don't like systemctl's auto-paging feature: -# export SYSTEMD_PAGER= - -# User specific aliases and functions -# If adding new functions to this file, note that you can add help text to the function -# by defining a variable with name __help containing the help text - -# Set up virtualenvwrapper -export WORKON_HOME=$HOME/.virtualenvs -export PIP_VIRTUALENV_BASE=$WORKON_HOME -export VIRTUALENV_USE_DISTRIBUTE=true -export PIP_RESPECT_VIRTUALENV=true -source /usr/bin/virtualenvwrapper.sh - -export HUBS_CONFIG={{ hubs_code_dir }}/config -export FLASK_APP={{ hubs_code_dir }}/hubs/app.py - - -hup() { - workon hubs - pushd {{ hubs_code_dir }} - FLASK_DEBUG=1 flask run --host 0.0.0.0 -} - -hreset() { - workon hubs - {% if hubs_db_type == "postgresql" %} - sudo -u postgres dropdb hubs - sudo -u postgres createdb -O hubs hubs - {% else %} - rm /var/lib/hubs/hubs.db - {% endif %} - rm /var/lib/hubs/cache.db - pushd {{ hubs_code_dir }} - python populate.py - popd - deactivate -} diff --git a/ansible/roles/hubs-dev/templates/fedmsg_config b/ansible/roles/hubs-dev/templates/fedmsg_config deleted file mode 100644 index c54fdb2..0000000 --- a/ansible/roles/hubs-dev/templates/fedmsg_config +++ /dev/null @@ -1,19 +0,0 @@ -config = { - - # Database - {% if hubs_db_type == "postgresql" %} - 'hubs.sqlalchemy.uri': 'postgresql://hubs:{{ hubs_db_password }}@localhost/hubs', - {% else %} - 'hubs.sqlalchemy.uri': 'sqlite:////var/lib/hubs/hubs.db', - {% endif %} - - # Some configuration for the general hubs cache. - "fedora-hubs.cache": { - "backend": "dogpile.cache.dbm", - #"expiration_time": 0, - "arguments": { - "filename": "/var/lib/hubs/cache.db", - }, - }, -} - diff --git a/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service b/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service deleted file mode 100644 index cc2b96e..0000000 --- a/ansible/roles/hubs-dev/templates/hubs-fedmsg-hub.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Hubs-specific fedmsg processing hub -After=network.target -Documentation=https://fedmsg.readthedocs.org/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub -WorkingDirectory={{ hubs_code_dir }} -Type=simple -User=fedmsg -Group=fedmsg -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-triage@.service b/ansible/roles/hubs-dev/templates/hubs-triage@.service deleted file mode 100644 index fc74242..0000000 --- a/ansible/roles/hubs-dev/templates/hubs-triage@.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=fedora-hubs triage worker #%i -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-webapp.service b/ansible/roles/hubs-dev/templates/hubs-webapp.service deleted file mode 100644 index 00ba431..0000000 --- a/ansible/roles/hubs-dev/templates/hubs-webapp.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=fedora-hubs frontend webapp -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:80 --threads 12 --log-config logging.ini --reload hubs.app:app -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs-worker@.service b/ansible/roles/hubs-dev/templates/hubs-worker@.service deleted file mode 100644 index 8073577..0000000 --- a/ansible/roles/hubs-dev/templates/hubs-worker@.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=fedora-hubs cache worker #%i -After=network.target -Documentation=https://pagure.io/fedora-hubs/ - -[Service] -ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config -Type=simple -User={{ main_user }} -Group={{ main_user }} -Restart=on-failure - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/hubs-dev/templates/hubs_config b/ansible/roles/hubs-dev/templates/hubs_config deleted file mode 100644 index d5a5394..0000000 --- a/ansible/roles/hubs-dev/templates/hubs_config +++ /dev/null @@ -1,8 +0,0 @@ -# Enter any hubs configuration here - -SECRET_KEY = "{{ hubs_secret_key }}" - -{% if hubs_dev_mode %} -# Allow the cookie to be sent of http since we work on localhost -OIDC_ID_TOKEN_COOKIE_SECURE = False -{% endif %} diff --git a/ansible/roles/hubs/defaults/main.yml b/ansible/roles/hubs/defaults/main.yml new file mode 100644 index 0000000..8f50281 --- /dev/null +++ b/ansible/roles/hubs/defaults/main.yml @@ -0,0 +1,8 @@ +main_user: hubs +hubs_dev_mode: yes +hubs_secret_key: changeme +hubs_code_dir: "/home/{{ main_user }}/devel" +hubs_venv_dir: "/home/{{ main_user }}/.virtualenvs/hubs" +hubs_db_type: sqlite +hubs_db_password: changeme +hubs_url: http://localhost:5000 diff --git a/ansible/roles/hubs/files/pg_hba.conf b/ansible/roles/hubs/files/pg_hba.conf new file mode 100644 index 0000000..c703fd9 --- /dev/null +++ b/ansible/roles/hubs/files/pg_hba.conf @@ -0,0 +1,14 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 md5 +# IPv6 local connections: +host all all ::1/128 md5 diff --git a/ansible/roles/hubs/handlers/main.yml b/ansible/roles/hubs/handlers/main.yml new file mode 100644 index 0000000..5d4ca3e --- /dev/null +++ b/ansible/roles/hubs/handlers/main.yml @@ -0,0 +1,5 @@ +- name: restart postgresql + service: name=postgresql state=restarted + +- name: restart fedmsg-hub + service: name=fedmsg-hub state=restarted diff --git a/ansible/roles/hubs/tasks/db-postgresql.yml b/ansible/roles/hubs/tasks/db-postgresql.yml new file mode 100644 index 0000000..d228b80 --- /dev/null +++ b/ansible/roles/hubs/tasks/db-postgresql.yml @@ -0,0 +1,54 @@ +# Set up Postgres, create the database, and populate it. + +- name: Install dependencies + dnf: name={{ item }} state=present + with_items: + - postgresql-server + - python-psycopg2 + +- name: Set up postgresql database + command: postgresql-setup --initdb + args: + creates: /var/lib/pgsql/data/base + +- name: Set up postgresql access rules to allow local access + copy: + src: pg_hba.conf + dest: /var/lib/pgsql/data/pg_hba.conf + owner: postgres + group: postgres + mode: 0600 + notify: restart postgresql + +- name: Start and enable postgresql + service: name=postgresql state=started enabled=yes + +- name: Set up the DB user + postgresql_user: + name: hubs + password: "{{ hubs_db_password }}" + role_attr_flags: NOSUPERUSER,NOCREATEROLE,NOCREATEDB + become_user: postgres + +- name: Create the database + postgresql_db: + name: hubs + owner: hubs + register: db_creation + become_user: postgres + +- name: Ease local access to the database + copy: + content: "*:*:hubs:hubs:{{ hubs_db_password }}" + dest: /home/{{ main_user }}/.pgpass + mode: 600 + owner: "{{ main_user }}" + group: "{{ main_user }}" + +- name: Populate the Fedora Hubs database + command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + args: + chdir: "{{ hubs_code_dir }}" + become_user: "{{ main_user }}" + when: db_creation|succeeded + diff --git a/ansible/roles/hubs/tasks/db-sqlite.yml b/ansible/roles/hubs/tasks/db-sqlite.yml new file mode 100644 index 0000000..120e1c7 --- /dev/null +++ b/ansible/roles/hubs/tasks/db-sqlite.yml @@ -0,0 +1,6 @@ +- name: Create and populate the Fedora Hubs database + command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" + become_user: "{{ main_user }}" + args: + creates: /var/lib/hubs/hubs.db + chdir: "{{ hubs_code_dir }}" diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml new file mode 100644 index 0000000..de8f328 --- /dev/null +++ b/ansible/roles/hubs/tasks/main.yml @@ -0,0 +1,168 @@ +--- +- name: Install helpful development packages + dnf: name={{ item }} state=present + with_items: + - git + - vim-enhanced + +- name: Install external dependencies + dnf: name={{ item }} state=present + with_items: + - npm + - redis + - fedmsg-hub + +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - python-sphinx + - python-virtualenvwrapper + - python2-devel + - python3-devel + - redhat-rpm-config + - sqlite-devel + +- name: Install the distribution versions of requirements.txt + dnf: name={{ item }} state=present + with_items: + - python-alembic + - python-arrow + - python-bleach + - python-decorator + - python-dogpile-cache + - python-fedmsg-core + - python-fedmsg-meta-fedora-infrastructure + - python-flask + - python-flask-oidc + - python-fmn-lib + - python-fmn-rules + - python-gunicorn + - python-futures + - python-html5lib + - python-munch + - pytz + - python-sqlalchemy + - python-markdown + - python2-pkgwat-api + - python-six + - python-pygments + - python-pygments-markdown-lexer + - python-retask + + +# Add various helpful configuration files +- name: Install a custom bashrc + template: src=bashrc dest=/home/{{ main_user }}/.bashrc + + +# Set up the Python development environment +- name: Install Fedora Hubs requirements.txt into hubs virtualenv + become_user: "{{ main_user }}" + pip: + requirements: "{{ hubs_code_dir }}/requirements.txt" + virtualenv: "{{ hubs_venv_dir}}" + virtualenv_site_packages: yes + +- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv + become_user: "{{ main_user }}" + pip: + requirements: "{{ hubs_code_dir }}/test-requirements.txt" + virtualenv: "{{ hubs_venv_dir}}" + virtualenv_site_packages: yes + +- name: Install other packages into hubs virtualenv + become_user: "{{ main_user }}" + pip: + name: "{{ item }}" + virtualenv: "{{ hubs_venv_dir }}" + virtualenv_site_packages: yes + with_items: + - bleach + - gunicorn + +#- name: Update httplib2 trust store +# copy: +# src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +# dest: /home/{{ main_user }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt +# remote_src: True + +- name: Add a basic Hubs configuration file + template: + src: "{{ item }}" + dest: "{{ hubs_code_dir }}/config" + with_first_found: + - hubs_config.{{ ansible_hostname }} + - hubs_config + become_user: "{{ main_user }}" + +- name: Add a basic fedmsg configuration file + template: + src: "{{ item }}" + dest: "/etc/fedmsg.d/hubs_config.py" + with_first_found: + - fedmsg_config.{{ ansible_hostname }} + - fedmsg_config + +- name: Create a directory for variable files + file: + path: "/var/lib/hubs" + state: directory + owner: "{{ main_user }}" + group: "{{ main_user }}" + mode: 755 + setype: httpd_sys_content_rw_t + +- name: Configure application to authenticate with iddev.fedorainfracloud.org + command: oidc-register https://iddev.fedorainfracloud.org/ {{hubs_url}} + become_user: "{{ main_user }}" + args: + creates: client_secrets.json + chdir: "{{ hubs_code_dir }}" + + +# Set up, create, and populate the database. +- include: db-{{ hubs_db_type }}.yml + + +# Set up JavaScript requirements +- name: Install npm packages + command: npm install + become_user: "{{ main_user }}" + args: + creates: node_modules + chdir: "{{ hubs_code_dir }}/hubs/static/client" + +- name: Build JavaScript assests + command: node_modules/.bin/webpack + become_user: "{{ main_user }}" + args: + chdir: "{{ hubs_code_dir }}/hubs/static/client" + +# Services +- name: Disable the system-wide fedmsg-hub + service: name=fedmsg-hub state=stopped enabled=no + +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-webapp + - hubs-triage@ + - hubs-worker@ + - hubs-fedmsg-hub + +- name: Start and enable the services + service: name={{ item }} state=started enabled=yes + with_items: + - redis + - hubs-webapp + - hubs-triage@1 + - hubs-triage@2 + - hubs-worker@1 + - hubs-worker@2 + - hubs-fedmsg-hub diff --git a/ansible/roles/hubs/templates/bashrc b/ansible/roles/hubs/templates/bashrc new file mode 100644 index 0000000..db0746d --- /dev/null +++ b/ansible/roles/hubs/templates/bashrc @@ -0,0 +1,45 @@ +# .bashrc + +# Source global definitions +if [ -f /etc/bashrc ]; then + . /etc/bashrc +fi + +# Uncomment the following line if you don't like systemctl's auto-paging feature: +# export SYSTEMD_PAGER= + +# User specific aliases and functions +# If adding new functions to this file, note that you can add help text to the function +# by defining a variable with name __help containing the help text + +# Set up virtualenvwrapper +export WORKON_HOME=$HOME/.virtualenvs +export PIP_VIRTUALENV_BASE=$WORKON_HOME +export VIRTUALENV_USE_DISTRIBUTE=true +export PIP_RESPECT_VIRTUALENV=true +source /usr/bin/virtualenvwrapper.sh + +export HUBS_CONFIG={{ hubs_code_dir }}/config +export FLASK_APP={{ hubs_code_dir }}/hubs/app.py + + +hup() { + workon hubs + pushd {{ hubs_code_dir }} + FLASK_DEBUG=1 flask run --host 0.0.0.0 +} + +hreset() { + workon hubs + {% if hubs_db_type == "postgresql" %} + sudo -u postgres dropdb hubs + sudo -u postgres createdb -O hubs hubs + {% else %} + rm /var/lib/hubs/hubs.db + {% endif %} + rm /var/lib/hubs/cache.db + pushd {{ hubs_code_dir }} + python populate.py + popd + deactivate +} diff --git a/ansible/roles/hubs/templates/fedmsg_config b/ansible/roles/hubs/templates/fedmsg_config new file mode 100644 index 0000000..c54fdb2 --- /dev/null +++ b/ansible/roles/hubs/templates/fedmsg_config @@ -0,0 +1,19 @@ +config = { + + # Database + {% if hubs_db_type == "postgresql" %} + 'hubs.sqlalchemy.uri': 'postgresql://hubs:{{ hubs_db_password }}@localhost/hubs', + {% else %} + 'hubs.sqlalchemy.uri': 'sqlite:////var/lib/hubs/hubs.db', + {% endif %} + + # Some configuration for the general hubs cache. + "fedora-hubs.cache": { + "backend": "dogpile.cache.dbm", + #"expiration_time": 0, + "arguments": { + "filename": "/var/lib/hubs/cache.db", + }, + }, +} + diff --git a/ansible/roles/hubs/templates/hubs-fedmsg-hub.service b/ansible/roles/hubs/templates/hubs-fedmsg-hub.service new file mode 100644 index 0000000..cc2b96e --- /dev/null +++ b/ansible/roles/hubs/templates/hubs-fedmsg-hub.service @@ -0,0 +1,15 @@ +[Unit] +Description=Hubs-specific fedmsg processing hub +After=network.target +Documentation=https://fedmsg.readthedocs.org/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub +WorkingDirectory={{ hubs_code_dir }} +Type=simple +User=fedmsg +Group=fedmsg +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs/templates/hubs-triage@.service b/ansible/roles/hubs/templates/hubs-triage@.service new file mode 100644 index 0000000..fc74242 --- /dev/null +++ b/ansible/roles/hubs/templates/hubs-triage@.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs triage worker #%i +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs/templates/hubs-webapp.service b/ansible/roles/hubs/templates/hubs-webapp.service new file mode 100644 index 0000000..00ba431 --- /dev/null +++ b/ansible/roles/hubs/templates/hubs-webapp.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs frontend webapp +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:80 --threads 12 --log-config logging.ini --reload hubs.app:app +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs/templates/hubs-worker@.service b/ansible/roles/hubs/templates/hubs-worker@.service new file mode 100644 index 0000000..8073577 --- /dev/null +++ b/ansible/roles/hubs/templates/hubs-worker@.service @@ -0,0 +1,16 @@ +[Unit] +Description=fedora-hubs cache worker #%i +After=network.target +Documentation=https://pagure.io/fedora-hubs/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker +WorkingDirectory={{ hubs_code_dir }} +Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Type=simple +User={{ main_user }} +Group={{ main_user }} +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/hubs/templates/hubs_config b/ansible/roles/hubs/templates/hubs_config new file mode 100644 index 0000000..d5a5394 --- /dev/null +++ b/ansible/roles/hubs/templates/hubs_config @@ -0,0 +1,8 @@ +# Enter any hubs configuration here + +SECRET_KEY = "{{ hubs_secret_key }}" + +{% if hubs_dev_mode %} +# Allow the cookie to be sent of http since we work on localhost +OIDC_ID_TOKEN_COOKIE_SECURE = False +{% endif %} diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index 623d093..d2a5e98 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -12,7 +12,7 @@ dest: /etc/sudoers.d/vagrant-nopasswd roles: - - role: hubs-dev + - role: hubs hubs_secret_key: vagrantmachinesecretkey hubs_db_type: sqlite hubs_url: http://localhost:5000 From 77155d0ac5d0792bc73d947e72d3cefc615dae57 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 20 2017 03:42:27 +0000 Subject: [PATCH 15/24] Missing dependency --- diff --git a/requirements.txt b/requirements.txt index 1eb3902..e2a8eb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +alembic arrow bleach<2.0.0 blinker From f7eb2655e9de1609057c96aac984e4cf052cb6a5 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 20 2017 04:21:12 +0000 Subject: [PATCH 16/24] Gunicorn can't listen on a privileged port --- diff --git a/ansible/roles/hubs/templates/hubs-webapp.service b/ansible/roles/hubs/templates/hubs-webapp.service index 00ba431..3ed4220 100644 --- a/ansible/roles/hubs/templates/hubs-webapp.service +++ b/ansible/roles/hubs/templates/hubs-webapp.service @@ -4,7 +4,7 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:80 --threads 12 --log-config logging.ini --reload hubs.app:app +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:8000 --threads 12 --log-config logging.ini --reload hubs.app:app WorkingDirectory={{ hubs_code_dir }} Environment=HUBS_CONFIG={{ hubs_code_dir }}/config Type=simple From 63ca8580b71de56bf73e94daf7005ddf0d18f293 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 10:00:03 +0000 Subject: [PATCH 17/24] Use a F25 box for Vagrant --- diff --git a/Vagrantfile.example b/Vagrantfile.example index 7fff78d..c03dce5 100644 --- a/Vagrantfile.example +++ b/Vagrantfile.example @@ -4,7 +4,7 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "fedora/24-cloud-base" + config.vm.box = "fedora/25-cloud-base" # Forward traffic on the host to the development server on the guest config.vm.network "forwarded_port", guest: 5000, host: 5000 From 1436e2c56290107cda10522e37ecd99ad8ac57b3 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 10:10:29 +0000 Subject: [PATCH 18/24] Put the code, the config and the data in the FHS --- diff --git a/Vagrantfile.example b/Vagrantfile.example index c03dce5..5caa591 100644 --- a/Vagrantfile.example +++ b/Vagrantfile.example @@ -23,7 +23,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # If you would prefer to use NFS to share the directory uncomment this and configure NFS #config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false config.vm.synced_folder ".", "/vagrant", disabled: true - config.vm.synced_folder ".", "/home/vagrant/devel", type: "sshfs", sshfs_opts_append: "-o nonempty" + config.vm.synced_folder ".", "/srv/hubs/fedora-hubs", type: "sshfs", sshfs_opts_append: "-o nonempty" # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) # you can create a local directory and share it to the guest's DNF cache. The directory needs to diff --git a/ansible/files/motd b/ansible/files/motd index f3d92ab..6c75466 100644 --- a/ansible/files/motd +++ b/ansible/files/motd @@ -3,7 +3,7 @@ Welcome to the Fedora Hubs development environment! Here are some tips: -* The code for Fedora Hubs is located at ~/devel/ +* The code for Fedora Hubs is located at /srv/hubs/fedora-hubs * You can type `workon hubs` to enter a configured Python virtualenv diff --git a/ansible/roles/hubs/defaults/main.yml b/ansible/roles/hubs/defaults/main.yml index 8f50281..ecd3429 100644 --- a/ansible/roles/hubs/defaults/main.yml +++ b/ansible/roles/hubs/defaults/main.yml @@ -1,8 +1,11 @@ main_user: hubs hubs_dev_mode: yes hubs_secret_key: changeme -hubs_code_dir: "/home/{{ main_user }}/devel" -hubs_venv_dir: "/home/{{ main_user }}/.virtualenvs/hubs" +hubs_base_dir: "/srv/hubs" +hubs_code_dir: "{{ hubs_base_dir }}/fedora-hubs" +hubs_conf_dir: "{{ hubs_base_dir }}/config" +hubs_venv_dir: "{{ hubs_base_dir }}/venv" +hubs_var_dir: "{{ hubs_base_dir }}/var" hubs_db_type: sqlite hubs_db_password: changeme hubs_url: http://localhost:5000 diff --git a/ansible/roles/hubs/tasks/db-sqlite.yml b/ansible/roles/hubs/tasks/db-sqlite.yml index 120e1c7..ec27bb8 100644 --- a/ansible/roles/hubs/tasks/db-sqlite.yml +++ b/ansible/roles/hubs/tasks/db-sqlite.yml @@ -2,5 +2,5 @@ command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py" become_user: "{{ main_user }}" args: - creates: /var/lib/hubs/hubs.db + creates: "{{ hubs_var_dir }}/hubs.db" chdir: "{{ hubs_code_dir }}" diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml index de8f328..edeac23 100644 --- a/ansible/roles/hubs/tasks/main.yml +++ b/ansible/roles/hubs/tasks/main.yml @@ -11,6 +11,8 @@ - npm - redis - fedmsg-hub + - python-virtualenv + - python3-flask-oidc - name: Install Fedora Hubs development packages dnf: name={{ item }} state=present @@ -20,7 +22,6 @@ - libffi-devel - openssl-devel - python-sphinx - - python-virtualenvwrapper - python2-devel - python3-devel - redhat-rpm-config @@ -59,6 +60,22 @@ template: src=bashrc dest=/home/{{ main_user }}/.bashrc +# Create directory structure + +- name: Create the directory structure + file: + path: "{{ item.path }}" + state: directory + owner: "{{ main_user }}" + group: "{{ main_user }}" + mode: "{{ item.mode }}" + #setype: httpd_sys_content_rw_t + with_items: + - {path: "{{ hubs_base_dir }}", mode: 755} + - {path: "{{ hubs_conf_dir }}", mode: 750} + - {path: "{{ hubs_var_dir }}", mode: 755} + + # Set up the Python development environment - name: Install Fedora Hubs requirements.txt into hubs virtualenv become_user: "{{ main_user }}" @@ -84,6 +101,21 @@ - bleach - gunicorn +- name: Install Fedora Hubs into the virtualenv + become_user: "{{ main_user }}" + pip: + name: "{{ hubs_code_dir }}" + editable: true + virtualenv: "{{ hubs_venv_dir}}" + virtualenv_site_packages: yes + +- name: Set bin file context in the virtualenv + file: + path: "{{ hubs_venv_dir }}/bin" + state: directory + recurse: true + setype: bin_t + #- name: Update httplib2 trust store # copy: # src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem @@ -93,11 +125,12 @@ - name: Add a basic Hubs configuration file template: src: "{{ item }}" - dest: "{{ hubs_code_dir }}/config" + dest: "{{ hubs_conf_dir }}/hubs_config.py" with_first_found: - hubs_config.{{ ansible_hostname }} - hubs_config become_user: "{{ main_user }}" + notify: "hubs configuration change" - name: Add a basic fedmsg configuration file template: @@ -106,6 +139,7 @@ with_first_found: - fedmsg_config.{{ ansible_hostname }} - fedmsg_config + notify: "hubs configuration change" - name: Create a directory for variable files file: @@ -117,11 +151,13 @@ setype: httpd_sys_content_rw_t - name: Configure application to authenticate with iddev.fedorainfracloud.org - command: oidc-register https://iddev.fedorainfracloud.org/ {{hubs_url}} + command: + oidc-register + --output-file {{ hubs_conf_dir }}/client_secrets.json + https://iddev.fedorainfracloud.org/ {{ hubs_url }} become_user: "{{ main_user }}" args: - creates: client_secrets.json - chdir: "{{ hubs_code_dir }}" + creates: "{{ hubs_conf_dir }}/client_secrets.json" # Set up, create, and populate the database. diff --git a/ansible/roles/hubs/templates/bashrc b/ansible/roles/hubs/templates/bashrc index db0746d..8902729 100644 --- a/ansible/roles/hubs/templates/bashrc +++ b/ansible/roles/hubs/templates/bashrc @@ -12,32 +12,32 @@ fi # If adding new functions to this file, note that you can add help text to the function # by defining a variable with name __help containing the help text -# Set up virtualenvwrapper -export WORKON_HOME=$HOME/.virtualenvs -export PIP_VIRTUALENV_BASE=$WORKON_HOME -export VIRTUALENV_USE_DISTRIBUTE=true -export PIP_RESPECT_VIRTUALENV=true -source /usr/bin/virtualenvwrapper.sh - -export HUBS_CONFIG={{ hubs_code_dir }}/config + +export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py export FLASK_APP={{ hubs_code_dir }}/hubs/app.py +workon() { + [ "$1" == "hubs" ] || ( echo "No such virtualenv."; exit 1 ) + source {{ hubs_venv_dir }}/bin/activate + cd {{ hubs_code_dir }} +} + hup() { - workon hubs + source {{ hubs_venv_dir }}/bin/activate pushd {{ hubs_code_dir }} - FLASK_DEBUG=1 flask run --host 0.0.0.0 + FLASK_DEBUG=1 flask run --host 0.0.0.0 --port 5000 } hreset() { - workon hubs + source {{ hubs_venv_dir }}/bin/activate {% if hubs_db_type == "postgresql" %} sudo -u postgres dropdb hubs sudo -u postgres createdb -O hubs hubs {% else %} - rm /var/lib/hubs/hubs.db + rm {{ hubs_var_dir }}/hubs.db {% endif %} - rm /var/lib/hubs/cache.db + rm {{ hubs_var_dir }}/cache.db pushd {{ hubs_code_dir }} python populate.py popd diff --git a/ansible/roles/hubs/templates/fedmsg_config b/ansible/roles/hubs/templates/fedmsg_config index c54fdb2..fc15ac4 100644 --- a/ansible/roles/hubs/templates/fedmsg_config +++ b/ansible/roles/hubs/templates/fedmsg_config @@ -1,10 +1,10 @@ config = { - + # Database {% if hubs_db_type == "postgresql" %} 'hubs.sqlalchemy.uri': 'postgresql://hubs:{{ hubs_db_password }}@localhost/hubs', {% else %} - 'hubs.sqlalchemy.uri': 'sqlite:////var/lib/hubs/hubs.db', + 'hubs.sqlalchemy.uri': 'sqlite:///{{ hubs_var_dir }}/hubs.db', {% endif %} # Some configuration for the general hubs cache. @@ -12,7 +12,7 @@ config = { "backend": "dogpile.cache.dbm", #"expiration_time": 0, "arguments": { - "filename": "/var/lib/hubs/cache.db", + "filename": "{{ hubs_var_dir }}/cache.db", }, }, } diff --git a/ansible/roles/hubs/templates/hubs-fedmsg-hub.service b/ansible/roles/hubs/templates/hubs-fedmsg-hub.service index cc2b96e..ca56996 100644 --- a/ansible/roles/hubs/templates/hubs-fedmsg-hub.service +++ b/ansible/roles/hubs/templates/hubs-fedmsg-hub.service @@ -5,7 +5,6 @@ Documentation=https://fedmsg.readthedocs.org/ [Service] ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub -WorkingDirectory={{ hubs_code_dir }} Type=simple User=fedmsg Group=fedmsg diff --git a/ansible/roles/hubs/templates/hubs-triage@.service b/ansible/roles/hubs/templates/hubs-triage@.service index fc74242..06ccacc 100644 --- a/ansible/roles/hubs/templates/hubs-triage@.service +++ b/ansible/roles/hubs/templates/hubs-triage@.service @@ -5,8 +5,7 @@ Documentation=https://pagure.io/fedora-hubs/ [Service] ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py Type=simple User={{ main_user }} Group={{ main_user }} diff --git a/ansible/roles/hubs/templates/hubs-webapp.service b/ansible/roles/hubs/templates/hubs-webapp.service index 3ed4220..59586d3 100644 --- a/ansible/roles/hubs/templates/hubs-webapp.service +++ b/ansible/roles/hubs/templates/hubs-webapp.service @@ -4,9 +4,13 @@ After=network.target Documentation=https://pagure.io/fedora-hubs/ [Service] -ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/gunicorn -b 0.0.0.0:8000 --threads 12 --log-config logging.ini --reload hubs.app:app -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +ExecStart= \ + {{ hubs_venv_dir }}/bin/python \ + /usr/bin/gunicorn -b 127.0.0.1:8000 --threads 12 \ + --log-config {{ hubs_conf_dir }}/logging.ini \ + {% if hubs_dev_mode %}--reload{% endif %} \ + hubs.app:app +Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py Type=simple User={{ main_user }} Group={{ main_user }} diff --git a/ansible/roles/hubs/templates/hubs-worker@.service b/ansible/roles/hubs/templates/hubs-worker@.service index 8073577..8f597f1 100644 --- a/ansible/roles/hubs/templates/hubs-worker@.service +++ b/ansible/roles/hubs/templates/hubs-worker@.service @@ -5,8 +5,7 @@ Documentation=https://pagure.io/fedora-hubs/ [Service] ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker -WorkingDirectory={{ hubs_code_dir }} -Environment=HUBS_CONFIG={{ hubs_code_dir }}/config +Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py Type=simple User={{ main_user }} Group={{ main_user }} diff --git a/ansible/roles/hubs/templates/hubs_config b/ansible/roles/hubs/templates/hubs_config index d5a5394..d079537 100644 --- a/ansible/roles/hubs/templates/hubs_config +++ b/ansible/roles/hubs/templates/hubs_config @@ -6,3 +6,5 @@ SECRET_KEY = "{{ hubs_secret_key }}" # Allow the cookie to be sent of http since we work on localhost OIDC_ID_TOKEN_COOKIE_SECURE = False {% endif %} + +OIDC_CLIENT_SECRETS = "{{ hubs_conf_dir }}/client_secrets.json" diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index d2a5e98..342ff0d 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -20,3 +20,9 @@ tasks: - name: Install the message of the day copy: src=files/motd dest=/etc/motd + + - name: Make a symlink to the code + file: + state: link + path: /home/{{ main_user }}/devel + src: /srv/hubs/fedora-hubs From 503d4fb35d167b5cac34a49dbda41aef7892a838 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 10:50:26 +0000 Subject: [PATCH 19/24] Deploy the webserver config in prod mode only --- diff --git a/ansible/files/motd b/ansible/files/motd index 6c75466..fdd814a 100644 --- a/ansible/files/motd +++ b/ansible/files/motd @@ -7,7 +7,7 @@ Here are some tips: * You can type `workon hubs` to enter a configured Python virtualenv -* Run `hup` to start gunicorn +* Run `hup` to start the flask webserver * Run `hreset` to delete the database and repopulate it diff --git a/ansible/roles/hubs/defaults/main.yml b/ansible/roles/hubs/defaults/main.yml index ecd3429..3f5cb63 100644 --- a/ansible/roles/hubs/defaults/main.yml +++ b/ansible/roles/hubs/defaults/main.yml @@ -1,5 +1,5 @@ main_user: hubs -hubs_dev_mode: yes +hubs_dev_mode: false hubs_secret_key: changeme hubs_base_dir: "/srv/hubs" hubs_code_dir: "{{ hubs_base_dir }}/fedora-hubs" @@ -8,4 +8,5 @@ hubs_venv_dir: "{{ hubs_base_dir }}/venv" hubs_var_dir: "{{ hubs_base_dir }}/var" hubs_db_type: sqlite hubs_db_password: changeme -hubs_url: http://localhost:5000 +hubs_url_hostname: localhost +hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %} diff --git a/ansible/roles/hubs/files/logging.ini b/ansible/roles/hubs/files/logging.ini new file mode 100644 index 0000000..3512fa6 --- /dev/null +++ b/ansible/roles/hubs/files/logging.ini @@ -0,0 +1,23 @@ +# From https://docs.python.org/2/howto/logging.html +[loggers] +keys=root + +[handlers] +keys=console + +[formatters] +keys=simple + +[logger_root] +level=DEBUG +handlers=console + +[handler_console] +class=StreamHandler +level=DEBUG +formatter=simple +args=(sys.stdout,) + +[formatter_simple] +format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s +datefmt=%H:%M:%S diff --git a/ansible/roles/hubs/handlers/main.yml b/ansible/roles/hubs/handlers/main.yml index 5d4ca3e..62e0484 100644 --- a/ansible/roles/hubs/handlers/main.yml +++ b/ansible/roles/hubs/handlers/main.yml @@ -3,3 +3,6 @@ - name: restart fedmsg-hub service: name=fedmsg-hub state=restarted + +# Webserver +- include: webserver.yml diff --git a/ansible/roles/hubs/handlers/webserver.yml b/ansible/roles/hubs/handlers/webserver.yml new file mode 100644 index 0000000..e6cb871 --- /dev/null +++ b/ansible/roles/hubs/handlers/webserver.yml @@ -0,0 +1,7 @@ +- name: restart hubs webapp + service: name=hubs-webapp state=restarted + listen: "hubs configuration change" + when: not hubs_dev_mode + +- name: restart nginx + service: name=nginx state=restarted diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml index edeac23..43406e7 100644 --- a/ansible/roles/hubs/tasks/main.yml +++ b/ansible/roles/hubs/tasks/main.yml @@ -26,6 +26,7 @@ - python3-devel - redhat-rpm-config - sqlite-devel + when: hubs_dev_mode - name: Install the distribution versions of requirements.txt dnf: name={{ item }} state=present @@ -41,7 +42,6 @@ - python-flask-oidc - python-fmn-lib - python-fmn-rules - - python-gunicorn - python-futures - python-html5lib - python-munch @@ -58,6 +58,7 @@ # Add various helpful configuration files - name: Install a custom bashrc template: src=bashrc dest=/home/{{ main_user }}/.bashrc + when: hubs_dev_mode # Create directory structure @@ -99,7 +100,6 @@ virtualenv_site_packages: yes with_items: - bleach - - gunicorn - name: Install Fedora Hubs into the virtualenv become_user: "{{ main_user }}" @@ -178,16 +178,17 @@ args: chdir: "{{ hubs_code_dir }}/hubs/static/client" + # Services - name: Disable the system-wide fedmsg-hub service: name=fedmsg-hub state=stopped enabled=no + - name: Install the service files template: src: "{{ item }}.service" dest: /etc/systemd/system/{{ item }}.service with_items: - - hubs-webapp - hubs-triage@ - hubs-worker@ - hubs-fedmsg-hub @@ -196,9 +197,13 @@ service: name={{ item }} state=started enabled=yes with_items: - redis - - hubs-webapp - hubs-triage@1 - hubs-triage@2 - hubs-worker@1 - hubs-worker@2 - hubs-fedmsg-hub + + +# Webserver +- include: webserver.yml + when: not hubs_dev_mode diff --git a/ansible/roles/hubs/tasks/webserver.yml b/ansible/roles/hubs/tasks/webserver.yml new file mode 100644 index 0000000..f070549 --- /dev/null +++ b/ansible/roles/hubs/tasks/webserver.yml @@ -0,0 +1,78 @@ +# Webserver config + +- name: Install the webserver packages + dnf: name={{ item }} state=present + with_items: + - python-gunicorn + - nginx + - make + + +- name: Generate SSL certificate and key + shell: + echo -e "--\nSomeState\nSomeCity\nSomeOrganization\nSomeOrganizationalUnit\nlocalhost.localdomain\nroot@localhost.localdomain" + | openssl req -utf8 -newkey rsa:2048 + -keyout /etc/pki/tls/private/localhost.key + -nodes -x509 -days 365 + -out /etc/pki/tls/certs/localhost.crt + args: + creates: /etc/pki/tls/certs/localhost.crt + #creates: /etc/pki/tls/private/localhost.key + + +#- name: Generate SSL certificate +# shell: +# echo -e "--\nSomeState\nSomeCity\nSomeOrganization\nSomeOrganizationalUnit\nlocalhost.localdomain\nroot@localhost.localdomain" +# | make -C /etc/pki/tls/certs testcert +# args: +# creates: /etc/pki/tls/certs/localhost.crt + + +- name: Gunicorn logging configuration + copy: + src: logging.ini + dest: "{{ hubs_conf_dir }}/logging.ini" + owner: "{{ main_user }}" + group: "{{ main_user }}" + notify: + - restart hubs webapp + + +- name: Nginx configuration + template: + src: nginx.conf + dest: /etc/nginx/conf.d/hubs.conf + notify: + - restart nginx + + +- name: Nginx SSL configuration + template: + src: "{{ item }}" + dest: /etc/nginx/ssl_params + with_first_found: + - nginx_ssl_params.{{ ansible_hostname }} + - nginx_ssl_params + notify: + - restart nginx + +- name: Allow network connection for Nginx + seboolean: + name: httpd_can_network_connect + state: yes + persistent: yes + + +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-webapp + + +- name: Start and enable the services + service: name={{ item }} state=started enabled=yes + with_items: + - hubs-webapp + - nginx diff --git a/ansible/roles/hubs/templates/hubs_config b/ansible/roles/hubs/templates/hubs_config index d079537..fbaf61d 100644 --- a/ansible/roles/hubs/templates/hubs_config +++ b/ansible/roles/hubs/templates/hubs_config @@ -8,3 +8,4 @@ OIDC_ID_TOKEN_COOKIE_SECURE = False {% endif %} OIDC_CLIENT_SECRETS = "{{ hubs_conf_dir }}/client_secrets.json" +OIDC_OPENID_REALM = "{{ hubs_url }}/oidc_callback" diff --git a/ansible/roles/hubs/templates/nginx.conf b/ansible/roles/hubs/templates/nginx.conf new file mode 100644 index 0000000..b969aaa --- /dev/null +++ b/ansible/roles/hubs/templates/nginx.conf @@ -0,0 +1,56 @@ +upstream hubs { + # fail_timeout=0 means we always retry an upstream even if it failed + # to return a good HTTP response + + # for UNIX domain socket setups + #server unix:/tmp/gunicorn.sock fail_timeout=0; + + # for a TCP configuration + server 127.0.0.1:8000 fail_timeout=0; +} + +# Redirect cleartext traffic to HTTPS +server { + listen 80; + listen [::]:80; + server_name {{ hubs_url_hostname }}; + return 301 https://$server_name$request_uri; +} + +# Main server block +server { + listen 443 deferred; + listen [::]:443 deferred; + client_max_body_size 4G; + + server_name {{ hubs_url_hostname }}; + + include ssl_params; + keepalive_timeout 5; + + # path for static files + root {{ hubs_code_dir }}/hubs/static; + + location / { + # checks for static file, if not found proxy to app + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # enable this if and only if you use HTTPS + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + # we don't want nginx trying to do something clever with + # redirects, we set the Host: header above already. + proxy_redirect off; + # Allow long-running queries (SSE): + proxy_buffering off; + proxy_pass http://hubs; + } + + #error_page 500 502 503 504 /500.html; + #location = /500.html { + # root /path/to/app/current/public; + #} + } diff --git a/ansible/roles/hubs/templates/nginx_ssl_params b/ansible/roles/hubs/templates/nginx_ssl_params new file mode 100644 index 0000000..c6bc906 --- /dev/null +++ b/ansible/roles/hubs/templates/nginx_ssl_params @@ -0,0 +1,9 @@ +ssl on; +ssl_certificate /etc/pki/tls/certs/localhost.crt; +ssl_certificate_key /etc/pki/tls/private/localhost.key; + +ssl_session_timeout 5m; + +ssl_protocols SSLv2 SSLv3 TLSv1; +ssl_ciphers HIGH:!aNULL:!MD5; +ssl_prefer_server_ciphers on; diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index 342ff0d..6ad295f 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -15,7 +15,7 @@ - role: hubs hubs_secret_key: vagrantmachinesecretkey hubs_db_type: sqlite - hubs_url: http://localhost:5000 + hubs_dev_mode: true tasks: - name: Install the message of the day From af637b03c13a7d171242125ad03f13b943008da9 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 10:50:26 +0000 Subject: [PATCH 20/24] Restart services on config change --- diff --git a/ansible/roles/hubs/handlers/main.yml b/ansible/roles/hubs/handlers/main.yml index 62e0484..30ce20d 100644 --- a/ansible/roles/hubs/handlers/main.yml +++ b/ansible/roles/hubs/handlers/main.yml @@ -3,6 +3,15 @@ - name: restart fedmsg-hub service: name=fedmsg-hub state=restarted + listen: "hubs configuration change" + +- name: restart hubs triage + service: name=hubs-triage@* state=restarted + listen: "hubs configuration change" + +- name: restart hubs workers + service: name=hubs-worker@* state=restarted + listen: "hubs configuration change" # Webserver - include: webserver.yml From 2de9f4bfb0de8b798a2a180a49e856e19dd1203e Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 10:50:26 +0000 Subject: [PATCH 21/24] The fedmsg consumer can't get the defaults ...because it does not use the Flask function to get the configuration. So add them to the local config file. --- diff --git a/ansible/roles/hubs/templates/fedmsg_config b/ansible/roles/hubs/templates/fedmsg_config index fc15ac4..96a7ce8 100644 --- a/ansible/roles/hubs/templates/fedmsg_config +++ b/ansible/roles/hubs/templates/fedmsg_config @@ -15,5 +15,9 @@ config = { "filename": "{{ hubs_var_dir }}/cache.db", }, }, + + # Fedmsg hub consumer + 'hubs.consumer.enabled': True, + 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', } From 91f48087f3bc4f026c784ba2f21338beea15caca Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 11:06:58 +0000 Subject: [PATCH 22/24] fixup! Put the code, the config and the data in the FHS --- diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml index 43406e7..9b2c241 100644 --- a/ansible/roles/hubs/tasks/main.yml +++ b/ansible/roles/hubs/tasks/main.yml @@ -74,7 +74,7 @@ with_items: - {path: "{{ hubs_base_dir }}", mode: 755} - {path: "{{ hubs_conf_dir }}", mode: 750} - - {path: "{{ hubs_var_dir }}", mode: 755} + - {path: "{{ hubs_var_dir }}", mode: 750} # Set up the Python development environment @@ -105,7 +105,7 @@ become_user: "{{ main_user }}" pip: name: "{{ hubs_code_dir }}" - editable: true + extra_args: -e virtualenv: "{{ hubs_venv_dir}}" virtualenv_site_packages: yes @@ -141,15 +141,6 @@ - fedmsg_config notify: "hubs configuration change" -- name: Create a directory for variable files - file: - path: "/var/lib/hubs" - state: directory - owner: "{{ main_user }}" - group: "{{ main_user }}" - mode: 755 - setype: httpd_sys_content_rw_t - - name: Configure application to authenticate with iddev.fedorainfracloud.org command: oidc-register From 8b24f244d2b6be9a4cb0f9a978c7826f37669c65 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 12:33:41 +0000 Subject: [PATCH 23/24] Minor improvements to the playbooks --- diff --git a/ansible/roles/hubs/handlers/main.yml b/ansible/roles/hubs/handlers/main.yml index 30ce20d..cd2d639 100644 --- a/ansible/roles/hubs/handlers/main.yml +++ b/ansible/roles/hubs/handlers/main.yml @@ -1,8 +1,8 @@ - name: restart postgresql service: name=postgresql state=restarted -- name: restart fedmsg-hub - service: name=fedmsg-hub state=restarted +- name: restart the hubs-specific fedmsg-hub + service: name=hubs-fedmsg-hub state=restarted listen: "hubs configuration change" - name: restart hubs triage diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml index 9b2c241..02f14fc 100644 --- a/ansible/roles/hubs/tasks/main.yml +++ b/ansible/roles/hubs/tasks/main.yml @@ -103,11 +103,9 @@ - name: Install Fedora Hubs into the virtualenv become_user: "{{ main_user }}" - pip: - name: "{{ hubs_code_dir }}" - extra_args: -e - virtualenv: "{{ hubs_venv_dir}}" - virtualenv_site_packages: yes + command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}" + args: + creates: "{{ hubs_venv_dir }}/lib/python2.7/site-packages/fedora-hubs.egg-link" - name: Set bin file context in the virtualenv file: @@ -116,12 +114,6 @@ recurse: true setype: bin_t -#- name: Update httplib2 trust store -# copy: -# src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem -# dest: /home/{{ main_user }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt -# remote_src: True - - name: Add a basic Hubs configuration file template: src: "{{ item }}" @@ -168,6 +160,7 @@ become_user: "{{ main_user }}" args: chdir: "{{ hubs_code_dir }}/hubs/static/client" + creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js" # Services From 3aa165a145f9d3e61352fbc148b2b3b1c99a45ae Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jun 21 2017 13:21:58 +0000 Subject: [PATCH 24/24] Remove useless SSL params in Nginx Use the default protocols (don't add SSLv2) and remove commented out lines in the playbook. --- diff --git a/ansible/roles/hubs/tasks/webserver.yml b/ansible/roles/hubs/tasks/webserver.yml index f070549..234089f 100644 --- a/ansible/roles/hubs/tasks/webserver.yml +++ b/ansible/roles/hubs/tasks/webserver.yml @@ -17,15 +17,6 @@ -out /etc/pki/tls/certs/localhost.crt args: creates: /etc/pki/tls/certs/localhost.crt - #creates: /etc/pki/tls/private/localhost.key - - -#- name: Generate SSL certificate -# shell: -# echo -e "--\nSomeState\nSomeCity\nSomeOrganization\nSomeOrganizationalUnit\nlocalhost.localdomain\nroot@localhost.localdomain" -# | make -C /etc/pki/tls/certs testcert -# args: -# creates: /etc/pki/tls/certs/localhost.crt - name: Gunicorn logging configuration diff --git a/ansible/roles/hubs/templates/nginx_ssl_params b/ansible/roles/hubs/templates/nginx_ssl_params index c6bc906..d97125f 100644 --- a/ansible/roles/hubs/templates/nginx_ssl_params +++ b/ansible/roles/hubs/templates/nginx_ssl_params @@ -1,9 +1,3 @@ ssl on; ssl_certificate /etc/pki/tls/certs/localhost.crt; ssl_certificate_key /etc/pki/tls/private/localhost.key; - -ssl_session_timeout 5m; - -ssl_protocols SSLv2 SSLv3 TLSv1; -ssl_ciphers HIGH:!aNULL:!MD5; -ssl_prefer_server_ciphers on;