From ad61609e94adc42c75acf9fc8c87c9cb9c56dcea Mon Sep 17 00:00:00 2001
From: Tristan Cacqueray <tdecacqu@redhat.com>
Date: Jun 04 2019 05:11:28 +0000
Subject: fedora-messaging: bootstrap the services


---

diff --git a/fedora-messaging/README.md b/fedora-messaging/README.md
new file mode 100644
index 0000000..b9a9008
--- /dev/null
+++ b/fedora-messaging/README.md
@@ -0,0 +1,13 @@
+# Fedora messaging gateway
+
+## Design
+
+The fedora messaging gateway is composed of 3 components:
+
+- [zuul gateway](https://github.com/TristanCacqueray/zuul-gateway): trigger job.
+- Consumer: record AMQP message
+- Processor: trigger job and report completed jobs
+
+## Setup
+
+To setup or update the system, run the `deploy.yaml` playbook.
diff --git a/fedora-messaging/deploy.yaml b/fedora-messaging/deploy.yaml
new file mode 100644
index 0000000..80dd090
--- /dev/null
+++ b/fedora-messaging/deploy.yaml
@@ -0,0 +1,248 @@
+---
+- hosts: localhost
+  become: yes
+  gather_facts: no
+  vars:
+    ansible_python_interpreter: /usr/bin/python2
+    user: fedora-messaging-zuul
+    state_dir: /var/lib/fedora-messaging-zuul
+    lib_dir: /usr/libexec/fedora-messaging-zuul
+    gateway_version: a96f543cd02398ac582952a44563372442b13a41
+    gateway_sum: "sha256:3bbda75372c1cf54bf8a4dc810c0c5372256a28ef28b7412bf8014586f18382f"
+    container_image: "fedora-messaging:latest"
+    state: started
+    zuul_url: https://fedora.softwarefactory-project.io/zuul
+    zuul_tenant: fedora-staging
+    zuul_connection: amqp
+    zuul_gateway: https://fedora.softwarefactory-project.io/amqp
+
+  pre_tasks:
+    - name: Create service user
+      user:
+        name: "{{ user }}"
+        home: "{{ state_dir }}"
+      register: user_object
+    - name: Create state directories
+      file:
+        path: "{{ state_dir }}/{{ item }}"
+        state: directory
+        mode: 0755
+        owner: "{{ user }}"
+      with_items:
+        - ""
+        - new
+        - done
+        - trash
+        - error
+    - name: Create libexec directory
+      file:
+        path: "{{ lib_dir }}"
+        state: directory
+        mode: 0755
+    - name: Install dependencies
+      package:
+        name:
+          - rh-python35-python-gunicorn
+          - rh-python35-python-requests
+          - rh-python35-python-flask
+          - rh-python35-enable-py3
+          - buildah
+          - podman
+
+  handlers:
+    - name: Restart zuul-gateway
+      service:
+        name: fedora-messaging-gateway
+        state: restarted
+      when: state == "started"
+    - name: Restart fedora-messaging services
+      service:
+        name: "fedora-messaging-{{ item }}"
+        state: restarted
+      with_items:
+        - consumer
+        - processor
+      when: state == "started"
+    - name: Reload apache
+      service:
+        name: httpd
+        state: reloaded
+
+  tasks:
+    - name: Copy the script
+      copy:
+        src: script.py
+        dest: "{{ lib_dir }}/script.py"
+        mode: 0755
+      notify: Restart fedora-messaging services
+    - name: Setup fedora-messaging configuration
+      copy:
+        content: |
+          amqp_url = "amqps://fedora:@rabbitmq.fedoraproject.org/%2Fpublic_pubsub"
+
+          [tls]
+          ca_cert = "/etc/fedora-messaging/cacert.pem"
+          keyfile = "/etc/fedora-messaging/fedora-key.pem"
+          certfile = "/etc/fedora-messaging/fedora-cert.pem"
+
+          [client_properties]
+          app = "Fedora Messaging Zuul"
+
+          [exchanges."amq.topic"]
+          type = "topic"
+          durable = true
+          auto_delete = false
+          arguments = {}
+
+          [qos]
+          prefetch_size = 0
+          prefetch_count = 25
+
+          [log_config]
+          version = 1
+          disable_existing_loggers = true
+
+          [log_config.formatters.simple]
+          format = "[%(levelname)-7s %(name)s] - %(message)s"
+
+          [log_config.handlers.console]
+          class = "logging.StreamHandler"
+          formatter = "simple"
+          stream = "ext://sys.stdout"
+
+          [log_config.root]
+          level = "ERROR"
+          handlers = ["console"]
+        dest: "{{ lib_dir }}/fedora-messaging-zuul.toml"
+
+    - name: Deploy the zuul-gateway
+      block:
+        - name: Fetch the gateway.py script
+          get_url:
+            url: "https://raw.githubusercontent.com/TristanCacqueray/zuul-gateway/{{ gateway_version }}/gateway.py"
+            dest: "{{ lib_dir }}/gateway.py"
+            checksum: "{{ gateway_sum }}"
+          notify: Restart zuul-gateway
+        - name: Setup systemd service
+          copy:
+            content: |
+              [Unit]
+              Description=Zuul gateway API Service
+              After=syslog.target network.target
+
+              [Service]
+              Type=simple
+              User={{ user }}
+              Environment=ZUUL_URL={{ zuul_url }}
+              Environment=ZUUL_TENANT={{ zuul_tenant }}
+              Environment=ZUUL_CONNECTION={{ zuul_connection }}
+              SyslogIdentifier=fedora-messaging-gateway
+              EnvironmentFile=-/etc/opt/rh/rh-python35/sysconfig/enable-py3
+              ExecStart=/opt/rh/rh-python35/root/usr/bin/gunicorn \
+                --chdir / -b 0.0.0.0:9042 --pythonpath {{ lib_dir }} gateway:app
+
+              [Install]
+              WantedBy=multi-user.target
+            dest: /etc/systemd/system/fedora-messaging-gateway.service
+          register: _gateway_service
+        - name: Start the service
+          systemd:
+            name: fedora-messaging-gateway.service
+            daemon-reload: "{% if _gateway_service is changed %}yes{% else %}no{% endif %}"
+            state: "{{ state }}"
+            enabled: yes
+        - name: Setup gateway rewrite
+          copy:
+            content: |
+              ProxyPass /amqp/ http://127.0.0.1:9042/ retry=0
+              ProxyPassReverse /amqp/ http://127.0.0.1:9042/
+            dest: /etc/httpd/conf.d/amqp.conf
+          notify: Reload apache
+
+    - name: Deploy the fedora-messaging-consumer
+      block:
+        - name: Copy image file
+          copy:
+            content: |
+              FROM fedora
+              RUN dnf install -y fedora-messaging python3-requests koji
+              RUN useradd --uid {{ user_object.uid }} -m fedora-messaging-zuul
+              USER fedora-messaging-zuul
+            dest: "{{ lib_dir }}/Buildahfile"
+          register: _buildahfile
+          notify: Restart fedora-messaging services
+        - name: Build the image
+          command: "buildah bud -f Buildahfile -t {{ container_image }} {{ lib_dir }}"
+          when: _buildahfile is changed
+        - name: Setup systemd service
+          copy:
+            content: |
+              [Unit]
+              Description=Zuul messaging consumer
+              After=syslog.target network.target
+
+              [Service]
+              Type=simple
+              SyslogIdentifier=fedora-messaging-consumer
+              ExecStart=/usr/bin/podman run --name fedora-messaging-consumer \
+                -v {{ lib_dir }}:{{ lib_dir }} \
+                -v {{ state_dir }}:{{ state_dir }} \
+                --env PYTHONUNBUFFERED=1 \
+                --env PYTHONPATH={{ lib_dir }} \
+                --workdir {{ state_dir }} \
+                --user {{ user_object.uid }} \
+                --rm {{ container_image }} \
+                fedora-messaging --conf {{ lib_dir }}/fedora-messaging-zuul.toml consume --callback script:Consumer
+              ExecStop=-/usr/bin/podman stop fedora-messaging-consumer
+              Restart=always
+              RestartSec=3s
+
+              [Install]
+              WantedBy=multi-user.target
+            dest: /etc/systemd/system/fedora-messaging-consumer.service
+          register: _consumer_service
+        - name: Start the service
+          systemd:
+            name: fedora-messaging-consumer.service
+            daemon-reload: "{% if _consumer_service is changed %}yes{% else %}no{% endif %}"
+            state: "{{ state }}"
+            enabled: yes
+
+    - name: Deploy the fedora-messaging-processor
+      block:
+        - name: Setup systemd service
+          copy:
+            content: |
+              [Unit]
+              Description=Zuul messaging processor
+              After=syslog.target network.target
+
+              [Service]
+              Type=simple
+              SyslogIdentifier=fedora-messaging-processor
+              ExecStart=/usr/bin/podman run --name fedora-messaging-processor \
+                -v {{ lib_dir }}:{{ lib_dir }} \
+                -v {{ state_dir }}:{{ state_dir }} \
+                --env PYTHONUNBUFFERED=1 \
+                --env FEDORA_MESSAGING_CONF={{ lib_dir }}/fedora-messaging-zuul.toml \
+                --env "ZUUL_GATEWAY={{ zuul_gateway }}" \
+                --env "ZUUL_URL={{ zuul_url }}" \
+                --env "ZUUL_TENANT={{ zuul_tenant }}" \
+                --workdir {{ state_dir }} \
+                --user {{ user_object.uid }} \
+                --rm {{ container_image }} \
+                {{ lib_dir }}/script.py
+              ExecStop=-/usr/bin/podman stop fedora-messaging-processor
+              Restart=always
+              RestartSec=3s
+
+              [Install]
+              WantedBy=multi-user.target
+            dest: /etc/systemd/system/fedora-messaging-processor.service
+          register: _processor_service
+        - name: Start the service
+          systemd:
+            name: fedora-messaging-processor.service
+            daemon-reload: "{% if _processor_service is changed %}yes{% else %}no{% endif %}"
+            state: "{{ state }}"
+            enabled: yes
diff --git a/fedora-messaging/script.py b/fedora-messaging/script.py
new file mode 100755
index 0000000..9194c4c
--- /dev/null
+++ b/fedora-messaging/script.py
@@ -0,0 +1,211 @@
+#!/bin/env python3
+# Copyright 2019 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import subprocess
+import time
+import logging
+
+from os import environ
+from typing import Dict, List, Optional, Type
+from pathlib import Path
+
+import requests
+from fedora_messaging import api, message
+
+
+#################
+# Configuration #
+#################
+ZUUL_GATEWAY = environ.get(
+    "ZUUL_GATEWAY",
+    "http://fedora.softwarefactory-project.io:9042").rstrip('/')
+ZUUL_URL = environ.get(
+    "ZUUL_URL", "https://fedora.softwarefactory-project.io/zuul/").rstrip('/')
+ZUUL_TENANT = environ.get("ZUUL_TENANT", "fedora-staging")
+ZUUL_API = ZUUL_URL + "/api/tenant/" + ZUUL_TENANT
+ZUUL_WEB = ZUUL_URL + "/t/" + ZUUL_TENANT
+
+DEBUG = True
+logging.basicConfig(
+    format='[%(levelname)-7s %(name)s] - %(message)s',
+    level=logging.DEBUG if DEBUG else logging.INFO)
+log = logging.getLogger("FedoraMessagingZuul")
+
+
+##########################
+# AMQP Message Zuul Jobs #
+##########################
+class Job(object):
+    def __init__(self, event: Dict) -> None:
+        self.event = event
+        self.ref = event["msg_id"]
+
+    @staticmethod
+    def match(event: Dict) -> bool:
+        """Returns True when the event matches the job"""
+
+    def trigger(self) -> Dict:
+        """Returns the list of Zuul jobs and custom variables"""
+
+    def report(self, build: Dict) -> Dict:
+        """Returns the AMQP message body"""
+
+
+class RpmLint(Job):
+    topic = "org.fedoraproject.prod.buildsys.build.state.change"
+
+    @staticmethod
+    def match(event: Dict) -> bool:
+        """Return True when koji build status is COMPLETED"""
+        return event["msg"]["new"] == 1
+
+    def trigger(self) -> Dict:
+        def kojiBuildInfo(buildId: int) -> List[str]:
+            p = subprocess.Popen(["koji", "buildinfo", str(buildId)],
+                                 stdout=subprocess.PIPE)
+            stdout, _ = p.communicate()
+            stdout = stdout.decode('utf-8')
+            if "RPMs:" in stdout:
+                return list(map(lambda x: x.replace(
+                    '/mnt/koji/', 'https://kojipkgs.fedoraproject.org/'),
+                    stdout.split("RPMs:")[1].strip().split('\n')))
+            return []
+
+        buildId = self.event["msg"]["build_id"]
+        rpms = kojiBuildInfo(buildId)
+        if not rpms:
+            raise RuntimeError("No rpms found in kojibuild %d" % buildId)
+        nodeset = dict(nodes=[{"name": "container", "label": "runc-centos"}])
+        return dict(jobs=["rpm-lint"], vars=dict(rpms=rpms), nodeset=nodeset)
+
+    def report(self, build: Dict) -> Dict:
+        return dict(
+            topic='org.fedoraproject.test.resultsdb.result.new',
+            body=dict(
+                type=["koji_build"],
+                href=getBuildsetUrl("refs/pull/" + self.ref + "/head"),
+                id=self.event["msg"]["build_id"],
+                outcome=build["status"],
+                message=build["comment"]))
+
+
+Jobs = [RpmLint]
+
+
+def event2job(event: Dict) -> Optional[Type[Job]]:
+    jobs = [job for job in Jobs
+            if job.topic == event["topic"] and job.match(event)]
+    if not jobs:
+        return None
+    if len(jobs) > 1:
+        raise RuntimeError("Multiple job match event")
+    return jobs[0]
+
+
+#############################
+# Message Consumer Callback #
+#############################
+class Consumer(object):
+    """Record relevant AMQP message"""
+    def __call__(self, message):
+        msgid = message.body.get("msg_id")
+
+        if not msgid or not event2job(message.body):
+            # No job match the event
+            return
+
+        log.info("%s: received %s", msgid, message.body.get("topic"))
+        with open("new/{}.json".format(msgid), "w") as of:
+            of.write(json.dumps(message.body))
+
+
+#############################
+# Message Processor Service #
+#############################
+def getBuildsetUrl(ref: str) -> str:
+    # TODO: use buildset url when available (https://review.opendev.org/630079)
+    builds = requests.get(ZUUL_API + "/builds?ref=" + ref).json()
+    if not builds:
+        raise RuntimeError("Couldn't get buildset url of %s" % ref)
+    return ZUUL_WEB + "/build/" + builds[0]["uuid"]
+
+
+def isBuildCompleted(build: Dict) -> bool:
+    return "comment" in build and "status" in build
+
+
+def process(event: Dict, builds: Dict) -> str:
+    """Process an event and return the final directory name"""
+    jobtype = event2job(event)
+    if not jobtype:
+        return "trash"
+    job = jobtype(event)
+    # First check if event has a running build
+    build = builds.get(job.ref)
+    if build and isBuildCompleted(build):
+        msg = message.Message(**job.report(build))
+        log.info("%s: reporting %s", job.ref, msg.body)
+        try:
+            # TODO: need an AMQP account...
+            # api.publish(msg)
+            pass
+        except Exception:
+            log.exception("Couldn't publish message %s:", msg)
+            if DEBUG:
+                raise
+            return "error"
+        requests.delete(ZUUL_GATEWAY + "/jobs/" + job.ref)
+        return "done"
+    elif build is None:
+        trigger = job.trigger()
+        trigger['vars']['amqp'] = event['msg']
+        log.info("triggering %s for %s", trigger, job.ref)
+        jobsDef = [{job: dict(vars=trigger['vars'],
+                              nodeset=trigger['nodeset'])}
+                   for job in trigger['jobs']]
+        zuul = json.dumps([dict(project=dict(check=dict(jobs=jobsDef)))])
+        log.debug("zuul configuration: %s", zuul)
+        requests.post(ZUUL_GATEWAY + "/jobs/" + job.ref, data=zuul)
+    return ""
+
+
+def main():
+    """Loop over message in new directory and call the process method"""
+    while True:
+        events = [x for x in Path('new').iterdir() if x.suffix == '.json']
+        if events:
+            # Grab pending jobs
+            builds = requests.get(ZUUL_GATEWAY + "/jobs").json()
+            for event in events:
+                try:
+                    result = process(json.loads(event.read_text()), builds)
+                    if result:
+                        event.rename(Path(result) / event.name)
+                except RuntimeError as e:
+                    log.error("%s: %s", event.name, e)
+                    event.rename(Path('error') / event.name)
+                except Exception:
+                    log.exception("%s: something went wrong:", event.name)
+                    event.rename(Path('error') / event.name)
+                    if DEBUG:
+                        raise
+        else:
+            log.debug("no events...")
+        time.sleep(5)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/fedora-messaging/setup.py b/fedora-messaging/setup.py
new file mode 100644
index 0000000..2cb5ae5
--- /dev/null
+++ b/fedora-messaging/setup.py
@@ -0,0 +1,33 @@
+# Copyright 2019 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import setuptools
+
+setuptools.setup(
+    name="zuul_fedora_messaging",
+    version="0.0.1",
+    author="Tristan de Cacqueray",
+    author_email="tdecacqu@redhat.com",
+    description="Process fedora messaging event with Zuul",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    url="https://pagure.io/fedora-project-config",
+    py_modules=["script"],
+    entry_points=dict(console_scripts=["zuul-fedora-messaging=script:main"]),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: ASL 2.0 License",
+        "Operating System :: OS Independent",
+    ],
+)
diff --git a/fedora-messaging/tox.ini b/fedora-messaging/tox.ini
new file mode 100644
index 0000000..1afc694
--- /dev/null
+++ b/fedora-messaging/tox.ini
@@ -0,0 +1,11 @@
+[tox]
+envlist = py35
+
+[testenv]
+deps =
+    flake8
+    mypy
+whitelist_externals = bash
+commands =
+    flake8 script.py
+    mypy --ignore-missing-imports script.py