From 56f9c3226ab6bd5a19a3a9c19607fe5293bc3d88 Mon Sep 17 00:00:00 2001
From: Josef Skladanka <jskladan@redhat.com>
Date: Apr 22 2018 12:04:26 +0000
Subject: Ondemand task scheduling based on testcase


---

diff --git a/.gitignore b/.gitignore
index 28f1a4c..b6dac6a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@
 
 # pytest cache
 /testing/.cache
+.pytest_cache
 
 # virtualenv
 /env_trigger/
diff --git a/conf/trigger_rules.yml.example b/conf/trigger_rules.yml.example
index c11b09f..43515b5 100644
--- a/conf/trigger_rules.yml.example
+++ b/conf/trigger_rules.yml.example
@@ -63,3 +63,8 @@
     message_type: GitHubPullRequestOpened
   do:
     - {tasks: [task-mtf]}
+- when:
+    message_type: TaskotronResultMissing
+    testcase: {$regex: '/^dist.rpmgrill(\..+)*/'}
+  do:
+    - {tasks: [rpmgrill], arches: ${arches}}
diff --git a/fedmsg.d/taskotron-trigger.py b/fedmsg.d/taskotron-trigger.py
index 8c2afd0..6d2f77a 100644
--- a/fedmsg.d/taskotron-trigger.py
+++ b/fedmsg.d/taskotron-trigger.py
@@ -6,4 +6,5 @@ config = {
     'taskotron.cloudcomposecompletedconsumer.enabled': True,
     'taskotron.modulebuilddonejobconsumer.enabled': True,
     'taskotron.githubpropenedjobconsumer.enabled': True,
+    'taskotron.taskotronresultmissingjobconsumer.enabled': True,
 }
diff --git a/jobtriggers/jobtrigger.py b/jobtriggers/jobtrigger.py
index 5cfc1c1..389026d 100644
--- a/jobtriggers/jobtrigger.py
+++ b/jobtriggers/jobtrigger.py
@@ -92,6 +92,9 @@ class JobTrigger(object):
             else:
                 self.log.error("Rule has invalid `do` section: %r", rule)
 
+            if 'arches' in item:
+                task['arches'] = item['arches']
+
             tasks.append(task)
 
         return tasks
diff --git a/jobtriggers/mongoquery_string_template.py b/jobtriggers/mongoquery_string_template.py
index dc7a6d5..f23b17e 100644
--- a/jobtriggers/mongoquery_string_template.py
+++ b/jobtriggers/mongoquery_string_template.py
@@ -79,6 +79,12 @@ class MongoTemplate(string.Template):
                 val = mapping[named]
                 # We use this idiom instead of str() because the latter will
                 # fail if val is a Unicode containing non-ASCII characters.
+                # FIXME - list, tuple and dict value handling hotfixed and not properly tested
+                if isinstance(val, list) or isinstance(val, tuple):
+                    return "[%s]" % ",".join(val)
+                if isinstance(val, dict):
+                    items = ["%s: %s" % (k, v) for k, v in val.iteritems()]
+                    return "{%s}" % ",".join(items)
                 return '%s' % (val,)
             if mo.group('escaped') is not None:
                 return self.delimiter
diff --git a/jobtriggers/taskotron_result_missing_msg.py b/jobtriggers/taskotron_result_missing_msg.py
new file mode 100644
index 0000000..0cd2c55
--- /dev/null
+++ b/jobtriggers/taskotron_result_missing_msg.py
@@ -0,0 +1,45 @@
+import fedmsg.consumers
+
+from . import config
+from .jobtrigger import JobTrigger
+from . import exceptions as exc
+
+MESSAGE_TYPE = 'TaskotronResultMissing'
+
+class TaskotronResultMissingJobTrigger(JobTrigger):
+    def process(self, msg):
+        item = msg['msg']['item']
+        item_type = msg['msg']['item_type']
+        testcase = msg['msg']['testcase']
+        if 'arch' in msg['msg']:
+            arches = [msg['msg']['arch']]
+        else:
+            arches = config.valid_arches
+
+        data = {
+            "_msg": msg,
+            "message_type": MESSAGE_TYPE,
+            "item": item,
+            "item_type": item_type,
+            "testcase": testcase,
+            "arches": arches,
+        }
+        return data
+
+
+class TasktoronResultMissingJobConsumer(fedmsg.consumers.FedmsgConsumer):
+    topic = "org.fedoraproject.%s.taskotron.result_missing" % config.deployment_type
+    config_key = 'taskotron.taskotronresultmissingjobconsumer.enabled'
+
+    def __init__(self, *args, **kw):
+        super(TasktoronResultMissingJobConsumer, self).__init__(*args, **kw)
+        self.trigger = TaskotronResultMissingJobTrigger(self.log)
+
+    def consume(self, message):
+        msg = fedmsg.encoding.loads(message.body)
+        try:
+            data = self.trigger.process(msg)
+            self.trigger.do_trigger(data)
+        except exc.TriggerMsgError, e:
+            self.log.debug(e)
+            return
diff --git a/requirements.txt b/requirements.txt
index 7d87b34..e1aa418 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,6 +11,7 @@ fedmsg[consumers]
 mongoquery
 requests
 Twisted
+koji
 PyYAML >= 3.11
 
 # Test suite requirements
diff --git a/testing/test_jobtrigger.py b/testing/test_jobtrigger.py
index 9fa5921..6b3ed5f 100644
--- a/testing/test_jobtrigger.py
+++ b/testing/test_jobtrigger.py
@@ -81,6 +81,17 @@ class TestJobtrigger():
         assert tasks[0]['item_type'] == self.ref_message_data['item_type']
         assert tasks[0]['arches'] == self.ref_default_arches
 
+
+    def test__tasks_from_rule_with_arches(self):
+        rule = {'do': [{'tasks': self.ref_tasks, 'arches': ['bogusarch']}]}
+        tasks = self.helper._tasks_from_rule(rule, self.ref_message_data)
+        assert len(tasks) == 1
+        assert tasks[0]['tasks'] == self.ref_tasks
+        assert tasks[0]['item'] == self.ref_message_data['item']
+        assert tasks[0]['item_type'] == self.ref_message_data['item_type']
+        assert tasks[0]['arches'] == ['bogusarch']
+
+
     def test__tasks_from_rule_discover(self):
         ref_repo = 'http://bogus.repo/firefox.git'
         rule = {'do': [{'discover': {'repo': ref_repo}}]}
@@ -140,6 +151,34 @@ class TestJobtrigger():
         assert runner_calls[3][1] == (
             self.ref_item, self.ref_item_type, self.ref_tasks[1], self.ref_default_arches[1])
 
+    def test_do_trigger_runtasks_with_arches(self, monkeypatch):
+        ref_arches = ['arch_foo', 'arch_bar']
+        mock__load_rules = mock.Mock(return_value=[
+                {
+                    'do': [{'tasks': self.ref_tasks, 'arches': ref_arches}],
+                    'when': {'message_type': 'RunTasks'}
+                },
+            ])
+        monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules)
+
+        message_data = copy.deepcopy(self.ref_message_data)
+        message_data['message_type'] = 'RunTasks'
+
+        self.helper.do_trigger(message_data)
+
+        runner_calls = self.helper.runner.trigger_job.calls()
+
+        assert len(runner_calls) == 4
+
+        assert runner_calls[0][1] == (
+            self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[0])
+        assert runner_calls[1][1] == (
+            self.ref_item, self.ref_item_type, self.ref_tasks[0], ref_arches[1])
+        assert runner_calls[2][1] == (
+            self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[0])
+        assert runner_calls[3][1] == (
+            self.ref_item, self.ref_item_type, self.ref_tasks[1], ref_arches[1])
+
     def test_do_trigger_discover(self, monkeypatch):
         mock__load_rules = mock.Mock(return_value=self.ref_rules)
         monkeypatch.setattr(self.helper, '_load_rules', mock__load_rules)
diff --git a/testing/test_taskotron_result_missing_trigger.py b/testing/test_taskotron_result_missing_trigger.py
new file mode 100644
index 0000000..6902fa2
--- /dev/null
+++ b/testing/test_taskotron_result_missing_trigger.py
@@ -0,0 +1,68 @@
+import pytest
+from dingus import Dingus
+from munch import Munch
+from copy import deepcopy
+import fedmsg.encoding
+import fedmsg.consumers
+
+from jobtriggers import taskotron_result_missing_msg
+
+
+@pytest.mark.usefixtures('prepare')
+class TestTaskotronResultMissingJobConsumer():
+
+    @pytest.fixture
+    def prepare(self, monkeypatch):
+        self.ref_item = 'ref_item'
+        self.ref_item_type = 'ref_item_type'
+        self.ref_testcase = 'ref.testcase.subcase'
+        self.ref_arch = 'ref_arch'
+        self.ref_task = 'ref_task'
+        self.ref_validarches = ['i386', 'x86_64']
+
+        self._create_msg(
+            self.ref_item, self.ref_item_type, self.ref_testcase, self.ref_arch)
+
+        self.ref_data = {
+            "_msg": {},
+            "message_type": "TaskotronResultMissing",
+            "item": self.ref_item,
+            "item_type": self.ref_item_type,
+            "testcase": self.ref_testcase,
+            "arches": [self.ref_arch],
+        }
+
+        stub_hub = Munch(config=Munch(get=0))
+        self.helper = taskotron_result_missing_msg.TasktoronResultMissingJobConsumer(stub_hub)
+
+        self.helper.trigger.runner = Dingus()
+        taskotron_result_missing_msg.config.trigger_rules_template = """---
+- do:
+  - {tasks: [ref_task], arches: ${arches}}
+  when: {message_type: TaskotronResultMissing, testcase: {$regex: '/^ref.testcase(\..+)*/'}}
+"""
+
+        taskotron_result_missing_msg.config.valid_arches = self.ref_validarches
+        taskotron_result_missing_msg.config.job_logging = False
+
+    def _create_msg(self, ref_item, ref_item_type, ref_testcase, ref_arch):
+        self.ref_message = Munch(body='{"i": 1,\
+                                        "msg": {"item": "%s",\
+                                                "item_type": "%s",\
+                                                "testcase": "%s",\
+                                                "arch": "%s"},\
+                                        "timestamp": 1359603469.21164,\
+                                        "topic": "org.fedoraproject.prod.taskotron.result_missing",\
+                                        "username": "apache"}' %
+                                 (ref_item, ref_item_type, ref_testcase, ref_arch))
+
+    def test_consume(self):
+        self.helper.consume(self.ref_message)
+
+        runner_calls = self.helper.trigger.runner.trigger_job.calls()
+
+        assert len(runner_calls) == 1
+
+        assert runner_calls[0][1] == (self.ref_item, self.ref_item_type,
+                                      self.ref_task, self.ref_arch)
+

