From 1c87b6faa738873daecd08033b94eae830b973d2 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard <aurelien@bompard.org>
Date: Feb 26 2018 18:24:53 +0000
Subject: Protect widgets against unavailable backend server


---

diff --git a/hubs/tests/widgets/test_badges.py b/hubs/tests/widgets/test_badges.py
index c8b2a6d..647d6c5 100644
--- a/hubs/tests/widgets/test_badges.py
+++ b/hubs/tests/widgets/test_badges.py
@@ -72,7 +72,7 @@ class TestBadges(WidgetTest):
         widget = widget_instance('ralph', self.plugin)
         func = widget.module.get_cached_functions()['GetBadges']
         result = func(widget).execute()
-        self.assertEqual(result, {})
+        self.assertEqual(result, None)
 
     @patch("hubs.widgets.badges.requests")
     def test_malformed_data(self, requests):
@@ -82,4 +82,4 @@ class TestBadges(WidgetTest):
         widget = widget_instance('ralph', self.plugin)
         func = widget.module.get_cached_functions()['GetBadges']
         result = func(widget).execute()
-        self.assertEqual(result, {})
+        self.assertEqual(result, None)
diff --git a/hubs/utils/datagrepper.py b/hubs/utils/datagrepper.py
index 0a266aa..adc37e0 100644
--- a/hubs/utils/datagrepper.py
+++ b/hubs/utils/datagrepper.py
@@ -36,7 +36,7 @@ def datagrepper_get(args):
             log.warning(
                 "Datagrepper request timed out (args: %s)", paged_args)
             return None
-        if response.status_code != 200:
+        if not response.ok:
             log.warning(
                 "Datagrepper request failed with args %s: %s",
                 paged_args, response.text)
diff --git a/hubs/utils/github.py b/hubs/utils/github.py
index 148efa6..0b374c9 100644
--- a/hubs/utils/github.py
+++ b/hubs/utils/github.py
@@ -21,7 +21,10 @@ def github_repo_is_valid(username, repo):
     log.info("Finding github repo for {} and {} ".format(repo, username))
     tmpl = "https://api.github.com/repos/{username}/{repo}"
     url = tmpl.format(username=username, repo=repo)
-    result = requests.get(url, timeout=5)
+    try:
+        result = requests.get(url, timeout=5)
+    except requests.exceptions.RequestException:
+        return False
     return result.ok
 
 
diff --git a/hubs/widgets/badges/__init__.py b/hubs/widgets/badges/__init__.py
index 344048b..7f6c0d5 100644
--- a/hubs/widgets/badges/__init__.py
+++ b/hubs/widgets/badges/__init__.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import logging
+
 import requests
 from collections import defaultdict, OrderedDict
 
@@ -8,6 +10,9 @@ from hubs.widgets.view import RootWidgetView
 from hubs.widgets.caching import CachedFunction
 
 
+log = logging.getLogger('hubs.widgets')
+
+
 class Badges(Widget):
     name = "badges"
     position = "right"
@@ -24,7 +29,13 @@ class BaseView(RootWidgetView):
     def get_context(self, instance, *args, **kwargs):
         context = {}
         get_badges = GetBadges(instance)
-        context.update(get_badges())
+        badges = get_badges()
+        if badges is None:
+            return dict(
+                failure="Could not connect to the badges server, please try "
+                        "again later."
+            )
+        context.update(badges)
         return context
 
 
@@ -34,12 +45,21 @@ class GetBadges(CachedFunction):
         username = self.instance.hub.name
         url = "https://badges.fedoraproject.org/user/{username}/json"
         url = url.format(username=username)
-        response = requests.get(url)
+        try:
+            response = requests.get(url)
+        except requests.exceptions.RequestException as e:
+            log.warning(
+                "Badges widget: could not get URL {}: {}".format(url, e),
+                exc_info=True)
+            return None
         try:
             response = response.json()
             assertions = response['assertions']
-        except (KeyError, ValueError):
-            return {}
+        except (KeyError, ValueError) as e:
+            log.warning(
+                "Badges widget: could not read response from URL "
+                "{}: {}".format(url, e), exc_info=True)
+            return None
         tag_totals = defaultdict(int)
         latest_badge = None
         for badge in assertions:
@@ -67,6 +87,9 @@ class GetBadges(CachedFunction):
                     badge_percentage=round(response["percent_earned"], 1)
         )
 
+    def _should_cache(self, value):
+        return value is not None
+
     def should_invalidate(self, message):
         if message['topic'].endswith('hubs.widget.update'):
             if message['msg']['widget']['id'] != self.instance.idx:
diff --git a/hubs/widgets/badges/templates/root.html b/hubs/widgets/badges/templates/root.html
index 94d5e34..d381718 100644
--- a/hubs/widgets/badges/templates/root.html
+++ b/hubs/widgets/badges/templates/root.html
@@ -1,3 +1,8 @@
+{% if failure %}
+  <p class="text-warning mb-0">
+    <em>{{ failure }}</em>
+  </p>
+{% else %}
 <div class="row mb-2">
   <div class="col-6">
     <h6>Badges Earned</h6>
@@ -39,6 +44,7 @@
     </ul>
 </div>
 </div>
+{% endif %}
 
 <style>
 .widget-badges h6 {
diff --git a/hubs/widgets/halp/functions.py b/hubs/widgets/halp/functions.py
index d9ab2a8..920599b 100644
--- a/hubs/widgets/halp/functions.py
+++ b/hubs/widgets/halp/functions.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals, absolute_import
 
-
 import datetime
 import logging
 import re
@@ -38,6 +37,9 @@ class GetRequests(CachedFunction):
             return None
         return [_format_message(msg) for msg in raw_messages]
 
+    def _should_cache(self, value):
+        return value is not None
+
     def should_invalidate(self, message):
         if message["topic"] != self.TOPIC:
             return False
diff --git a/hubs/widgets/mailinglist/__init__.py b/hubs/widgets/mailinglist/__init__.py
index faae5a6..6608f45 100644
--- a/hubs/widgets/mailinglist/__init__.py
+++ b/hubs/widgets/mailinglist/__init__.py
@@ -1,5 +1,9 @@
 from __future__ import unicode_literals
 
+import logging
+
+import requests
+
 from hubs.utils import hyperkitty
 from hubs.utils.fedmsg import get_fedmsg_config
 from hubs.widgets.base import Widget
@@ -9,6 +13,9 @@ from hubs.widgets.caching import CachedFunction
 fedmsg_config = get_fedmsg_config()
 
 
+log = logging.getLogger('hubs.widgets')
+
+
 class MailingList(Widget):
 
     name = "mailinglist"
@@ -29,7 +36,11 @@ class BaseView(RootWidgetView):
         if not instance.hub.config["mailing_list"]:
             return dict(no_list_defined=True)
         get_threads = GetMailingListThreads(instance)
-        return get_threads()
+        threads = get_threads()
+        if threads is None:
+            return dict(failure="Could not contact the mailing list server, "
+                                "please try again later.")
+        return threads
 
 
 class GetMailingListThreads(CachedFunction):
@@ -37,8 +48,15 @@ class GetMailingListThreads(CachedFunction):
     def execute(self):
         threads = list()
         listname = self.instance.hub.config["mailing_list"]
-        for t in hyperkitty.last_ten_threads(listname):
-            threads.append(t)
+        try:
+            for t in hyperkitty.last_ten_threads(listname):
+                threads.append(t)
+        except requests.exceptions.RequestException as e:
+            log.warning(
+                "HyperKitty widget: could not get data from HyperKitty: %s",
+                e, exc_info=True
+            )
+            return None
         new_thread_url = "/".join([hyperkitty.HYPERKITTY_URL,
                                    "list",
                                    listname,
@@ -47,6 +65,9 @@ class GetMailingListThreads(CachedFunction):
         return dict(threads=threads,
                     new_thread_url=new_thread_url)
 
+    def _should_cache(self, value):
+        return value is not None
+
     def should_invalidate(self, message):
         if ".mailman.receive" in message["topic"]:
             configlistname = self.instance.hub.config["mailing_list"]
diff --git a/hubs/widgets/mailinglist/templates/root.html b/hubs/widgets/mailinglist/templates/root.html
index bdf28af..123573b 100644
--- a/hubs/widgets/mailinglist/templates/root.html
+++ b/hubs/widgets/mailinglist/templates/root.html
@@ -10,6 +10,10 @@
   <p class="text-warning mb-0">
     <em>You must configure a mailinglist in the hub configuration in order to use this widget.</em>
   </p>
+{% elif failure %}
+  <p class="text-warning mb-0">
+    <em>{{ failure }}</em>
+  </p>
 {% else %}
   {% for thread in threads[:5] %}
     <div class="row py-2">
diff --git a/hubs/widgets/meetings/__init__.py b/hubs/widgets/meetings/__init__.py
index 9c9c92e..f1d0fb4 100644
--- a/hubs/widgets/meetings/__init__.py
+++ b/hubs/widgets/meetings/__init__.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
-import collections
 import datetime
+import logging
+from collections import defaultdict, OrderedDict as ordereddict
+
 import requests
-from collections import OrderedDict as ordereddict
 
 import hubs.models
 from hubs.utils import validators
@@ -13,6 +14,9 @@ from hubs.widgets.view import RootWidgetView
 from hubs.widgets.caching import CachedFunction
 
 
+log = logging.getLogger('hubs.widgets')
+
+
 class Meetings(Widget):
 
     name = "meetings"
@@ -82,9 +86,21 @@ class GetMeetings(CachedFunction):
             base = ('https://apps.fedoraproject.org/calendar/api/meetings/'
                     '?calendar=%s')
             url = base % c
-            response = requests.get(url).json()
-
-            tmp = collections.defaultdict(list)
+            try:
+                response = requests.get(url).json()
+            except requests.exceptions.RequestException as e:
+                log.warning(
+                    "Meetings widget: could not get URL %s: %s",
+                    url, e, exc_info=True
+                )
+                continue
+            except ValueError as e:
+                log.warning(
+                    "Meetings widget: could not read from URL %s: %s", url, e
+                )
+                continue
+
+            tmp = defaultdict(list)
             for meeting in response['meetings']:
                 if meeting.get('meeting_information_html'):
                     meeting['meeting_information_html'] = markup(
diff --git a/hubs/widgets/newestissues/__init__.py b/hubs/widgets/newestissues/__init__.py
index eab5a53..b05dfdc 100644
--- a/hubs/widgets/newestissues/__init__.py
+++ b/hubs/widgets/newestissues/__init__.py
@@ -1,5 +1,10 @@
 from __future__ import unicode_literals
 
+import logging
+
+import requests
+
+
 from hubs.utils import pagure
 from hubs.utils.github import github_issues
 from hubs.utils.fedmsg import get_fedmsg_config
@@ -10,6 +15,9 @@ from hubs.widgets.caching import CachedFunction
 fedmsg_config = get_fedmsg_config()
 
 
+log = logging.getLogger('hubs.widgets')
+
+
 class NewestIssues(Widget):
 
     name = "newestissues"
@@ -46,7 +54,17 @@ class GetPagureIssues(CachedFunction):
         for repo in self.instance.hub.config["pagure"]:
             try:
                 issues = pagure.pagure_issues(repo)
-            except ValueError:
+            except ValueError as e:
+                log.warning(
+                    "NewestIssues widget: could not read Pagure issues: %s",
+                    e, exc_info=True
+                )
+                continue
+            except requests.exceptions.RequestException as e:
+                log.warning(
+                    "NewestIssues widget: could not get Pagure issues: %s",
+                    e, exc_info=True
+                )
                 continue
 
             for issue in issues:
@@ -83,8 +101,15 @@ class GetGithubIssues(CachedFunction):
         all_issues = list()
 
         for repo in self.instance.hub.config["github"]:
-            for issue in github_issues(repo, token=None):
-                all_issues.append(issue)
+            try:
+                for issue in github_issues(repo, token=None):
+                    all_issues.append(issue)
+            except IOError as e:
+                log.warning(
+                    "NewestIssues widget: could not read Github issues: %s",
+                    e, exc_info=True
+                )
+                continue
 
         return all_issues
 
diff --git a/hubs/widgets/pullrequests/__init__.py b/hubs/widgets/pullrequests/__init__.py
index 929f870..2a50a61 100644
--- a/hubs/widgets/pullrequests/__init__.py
+++ b/hubs/widgets/pullrequests/__init__.py
@@ -1,5 +1,9 @@
 from __future__ import unicode_literals
 
+import logging
+
+import requests
+
 from hubs.utils import pagure
 from hubs.utils.github import github_pulls
 from hubs.utils.fedmsg import get_fedmsg_config
@@ -7,7 +11,8 @@ from hubs.widgets.base import Widget
 from hubs.widgets.view import RootWidgetView
 from hubs.widgets.caching import CachedFunction
 
-fedmsg_config = get_fedmsg_config()
+
+log = logging.getLogger('hubs.widgets')
 
 
 class PullRequests(Widget):
@@ -40,7 +45,17 @@ class GetPagurePullRequests(CachedFunction):
         for repo in self.instance.hub.config["pagure"]:
             try:
                 pulls = pagure.pagure_pulls(repo)
-            except ValueError:
+            except ValueError as e:
+                log.warning(
+                    "PullRequests widget: could not read Pagure PRs: %s",
+                    e, exc_info=True
+                )
+                continue
+            except requests.exceptions.RequestException as e:
+                log.warning(
+                    "PullRequests widget: could not get Pagure PRs: %s",
+                    e, exc_info=True
+                )
                 continue
 
             for pr in pulls:
@@ -76,11 +91,18 @@ class GetGithubPullRequests(CachedFunction):
     def execute(self):
         all_pr = list()
 
+        fedmsg_config = get_fedmsg_config()
         token = fedmsg_config.get('github.oauth_token')
         for repo in self.instance.hub.config["github"]:
-            for pr in github_pulls(repo, token=token):
-                all_pr.append(pr)
-
+            try:
+                for pr in github_pulls(repo, token=token):
+                    all_pr.append(pr)
+            except IOError as e:
+                log.warning(
+                    "PullRequests widget: could not get Github PRs: %s",
+                    e, exc_info=True
+                )
+                continue
         return all_pr
 
     def should_invalidate(self, message):
diff --git a/hubs/widgets/repositories/__init__.py b/hubs/widgets/repositories/__init__.py
index cd9732a..5714bae 100644
--- a/hubs/widgets/repositories/__init__.py
+++ b/hubs/widgets/repositories/__init__.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import logging
+
 import requests
 
 from hubs.utils import pagure
@@ -8,6 +10,9 @@ from hubs.widgets.view import RootWidgetView
 from hubs.widgets.caching import CachedFunction
 
 
+log = logging.getLogger('hubs.widgets')
+
+
 class Repositories(Widget):
 
     name = "repositories"
@@ -33,13 +38,35 @@ class GetRepos(CachedFunction):
         for repo in self.instance.hub.config["pagure"]:
             issues_url = '/'.join([pagure.PAGURE_URL, "api",
                                    "0", repo, "issues"])
-            issues_response = requests.get(issues_url)
-            issues_data = issues_response.json()
+            try:
+                issues_response = requests.get(issues_url)
+            except requests.exceptions.RequestException as e:
+                log.warning("Repositories widget: could not get URL %s: %s",
+                            issues_url, e, exc_info=True)
+                continue
+            try:
+                issues_data = issues_response.json()
+            except (TypeError, ValueError) as e:
+                log.warning(
+                    "Repositories widget: could not read response from URL "
+                    "%s: %s", issues_url, e, exc_info=True)
+                continue
 
             pr_url = '/'.join([pagure.PAGURE_URL, "api",
                                "0", repo, "pull-requests"])
-            pr_response = requests.get(pr_url)
-            pr_data = pr_response.json()
+            try:
+                pr_response = requests.get(pr_url)
+            except requests.exceptions.RequestException as e:
+                log.warning("Repositories widget: could not get URL %s: %s",
+                            pr_url, e, exc_info=True)
+                continue
+            try:
+                pr_data = pr_response.json()
+            except (TypeError, ValueError) as e:
+                log.warning(
+                    "Repositories widget: could not read response from URL "
+                    "%s: %s", pr_url, e, exc_info=True)
+                continue
 
             out = {"repo_name": repo,
                    "repo_url": "https://pagure.io/"+repo,
@@ -53,13 +80,35 @@ class GetRepos(CachedFunction):
             issues_url = ':'.join(
                 [('https://api.github.com/search/issues?'
                   'q=""+state:open+type:issue+repo'), repo])
-            issues_response = requests.get(issues_url)
-            issues_count = issues_response.json()['total_count']
+            try:
+                issues_response = requests.get(issues_url)
+            except requests.exceptions.RequestException as e:
+                log.warning("Repositories widget: could not get URL %s: %s",
+                            issues_url, e, exc_info=True)
+                continue
+            try:
+                issues_count = issues_response.json()['total_count']
+            except (TypeError, ValueError) as e:
+                log.warning(
+                    "Repositories widget: could not read response from URL "
+                    "%s: %s", issues_url, e, exc_info=True)
+                continue
             pr_url = ':'.join(
                 [('https://api.github.com/search/issues?'
                   'q=""+state:open+type:pr+repo'), repo])
-            pr_response = requests.get(pr_url)
-            pr_count = pr_response.json()['total_count']
+            try:
+                pr_response = requests.get(pr_url)
+            except requests.exceptions.RequestException as e:
+                log.warning("Repositories widget: could not get URL %s: %s",
+                            pr_url, e, exc_info=True)
+                continue
+            try:
+                pr_count = pr_response.json()['total_count']
+            except (TypeError, ValueError) as e:
+                log.warning(
+                    "Repositories widget: could not read response from URL "
+                    "%s: %s", pr_url, e, exc_info=True)
+                continue
 
             out = {"repo_name": repo,
                    "repo_url": "https://github.com/"+repo,
diff --git a/hubs/widgets/workflow/templates/updates2stable.html b/hubs/widgets/workflow/templates/updates2stable.html
index 5ba04b0..e1ddd20 100644
--- a/hubs/widgets/workflow/templates/updates2stable.html
+++ b/hubs/widgets/workflow/templates/updates2stable.html
@@ -1,4 +1,8 @@
-{% if updates %}
+{% if failure %}
+    <p class="text-warning mb-0">
+        <em>{{ failure }}</em>
+    </p>
+{% elif updates %}
     {% if session['nickname'] == username %}
     <a class="btn btn-success" target="_blank" href="https://admin.fedoraproject.org/updates/mine">Manage updates</a>
     <hr/>
diff --git a/hubs/widgets/workflow/updates2stable.py b/hubs/widgets/workflow/updates2stable.py
index 9720277..19360e2 100644
--- a/hubs/widgets/workflow/updates2stable.py
+++ b/hubs/widgets/workflow/updates2stable.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import logging
+
 import requests
 
 from hubs.widgets.base import Widget
@@ -7,6 +9,7 @@ from hubs.widgets.view import RootWidgetView
 from hubs.widgets.caching import CachedFunction
 
 
+log = logging.getLogger('hubs.widgets')
 # TODO - the bodhi api exposes a new flag we can use instead of scraping
 # comments...
 giveaway = 'can be pushed to stable now'
@@ -30,7 +33,12 @@ class BaseView(RootWidgetView):
         context = dict(
             username=instance.hub.name,
             )
-        context.update(get_pending())
+        pending = get_pending()
+        if pending is None:
+            context["failure"] = ("Could not get updates from Bodhi, please "
+                                  "try again later.")
+        else:
+            context.update(pending)
         return context
 
 
@@ -50,31 +58,36 @@ class GetPending(CachedFunction):
         url = bodhiurl + query
         headers = {'Accept': 'application/json'}
         response = requests.get(url, headers=headers)
-        if response.status_code == 200:
-            data = response.json()
-
-            # Limit this to only the updates that haven't already requested
-            # stable but which can actually be pushed to stable now.
-            data['updates'] = [
-                update for update in data['updates']
-                if update['request'] is None and
-                any([giveaway in comment['text']
-                     for comment in update['comments']])
-            ]
-
-            # Stuff some useful information in there.
-            baseurl = 'https://apps.fedoraproject.org/packages/images/icons/'
-            for update in data['updates']:
-                build = update['builds'][0]
-                nvr = build['nvr']
-                package = parse_nvr(nvr)
-                update['icon'] = baseurl + package[0] + '.png'
-                update['link'] = bodhiurl + update['title']
-
-            return data
-        else:
-            data = {}
-            return data
+        if not response.ok:
+            log.warning(
+                "Updates2Stable widget: Bodhi replied with status %s "
+                "on URL %s", response.status_code, url
+            )
+            return None
+
+        data = response.json()
+
+        # Limit this to only the updates that haven't already requested
+        # stable but which can actually be pushed to stable now.
+        data['updates'] = [
+            update for update in data['updates']
+            if update['request'] is None and
+            any([giveaway in comment['text']
+                 for comment in update['comments']])
+        ]
+
+        # Stuff some useful information in there.
+        baseurl = 'https://apps.fedoraproject.org/packages/images/icons/'
+        for update in data['updates']:
+            build = update['builds'][0]
+            nvr = build['nvr']
+            package = parse_nvr(nvr)
+            update['icon'] = baseurl + package[0] + '.png'
+            update['link'] = bodhiurl + update['title']
+        return data
+
+    def _should_cache(self, value):
+        return value is not None
 
     def should_invalidate(self, message):
         if not message['topic'].endswith('bodhi.update.comment'):