From d2fe6340381739cd4bd503c415b0829cf510c6ab Mon Sep 17 00:00:00 2001
From: Aurélien Bompard <aurelien@bompard.org>
Date: Aug 09 2017 15:17:24 +0000
Subject: [PATCH 1/2] Go through the Datagrepper pages


---

diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpFunctionsTestCase.test_execute b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpFunctionsTestCase.test_execute
index 1e03fdc..1dcff5f 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpFunctionsTestCase.test_execute
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpFunctionsTestCase.test_execute
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data
index 4d48985..645c292 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
@@ -103,7 +103,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data_wrong_hub b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data_wrong_hub
index 6b0a36a..73f7aba 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data_wrong_hub
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_data_wrong_hub
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_all b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_all
index dfd7580..87a393c 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_all
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_all
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
@@ -103,7 +103,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_date b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_date
index 49d31ac..496366e 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_date
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_date
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_hub b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_hub
index c2ca0a7..77ee8a3 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_hub
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_hub
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_meetingname b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_meetingname
index 07ed82f..b9c3aa6 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_meetingname
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_meetingname
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_people b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_people
index d3b8371..26609f7 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_people
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_people
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_requesters b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_requesters
index cd075ec..00b846c 100644
--- a/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_requesters
+++ b/hubs/tests/vcr-request-data/hubs.tests.widgets.test_halp.HalpViewsTestCase.test_search_requesters
@@ -7,7 +7,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
@@ -103,7 +103,7 @@ interactions:
       Connection: [keep-alive]
       User-Agent: [python-requests/2.13.0]
     method: GET
-    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000
+    uri: https://apps.fedoraproject.org/datagrepper/raw?topic=org.fedoraproject.prod.meetbot.meeting.item.help&delta=864000&page=1
   response:
     body:
       string: !!binary |
diff --git a/hubs/utils/datagrepper.py b/hubs/utils/datagrepper.py
new file mode 100644
index 0000000..42aa8ec
--- /dev/null
+++ b/hubs/utils/datagrepper.py
@@ -0,0 +1,51 @@
+from __future__ import unicode_literals, absolute_import
+
+import logging
+
+import requests
+
+from hubs.app import app
+
+log = logging.getLogger(__name__)
+
+
+def datagrepper_get(args):
+    """Make a query to Datagrepper, and return the ``raw_messages``.
+
+    This function will go through all pages to retrieve all the data.
+
+    Parameters:
+        args (dict): elements of the query string to Datagrepper.
+    """
+    url = "%s/raw" % app.config["DATAGREPPER_URI"]
+    result = []
+    page = 1
+    total_pages = 42  # it doesn't matter for now
+    log.debug("Getting data from Datagrepper: %s", str(args))
+    while page <= total_pages:
+        paged_args = args.copy()
+        paged_args["page"] = page
+        try:
+            # Call Datagrepper, but be careful to set a long timeout, this
+            # query may take some time to complete.
+            response = requests.get(url, paged_args, timeout=60)
+        except requests.exceptions.Timeout:
+            log.warning(
+                "Datagrepper request timed out (args: %s)", paged_args)
+            return None
+        if response.status_code != 200:
+            log.warning(
+                "Datagrepper request failed with args %s: %s",
+                paged_args, response.text)
+            return None
+        try:
+            response = response.json()
+        except ValueError as e:
+            log.warning(
+                "Datagrepper request with args %s could not be converted to "
+                "JSON: %s", paged_args, e)
+            return None
+        result.extend(response["raw_messages"])
+        total_pages = response["pages"]
+        page = page + 1
+    return result
diff --git a/hubs/widgets/halp/functions.py b/hubs/widgets/halp/functions.py
index 89cf0df..662081d 100644
--- a/hubs/widgets/halp/functions.py
+++ b/hubs/widgets/halp/functions.py
@@ -8,10 +8,10 @@ import time
 
 import flask
 import fedmsg.meta
-import requests
 from jinja2.utils import urlize
 
 from hubs.widgets.caching import CachedFunction
+from hubs.utils.datagrepper import datagrepper_get
 from .utils import find_hubs_for_msg
 
 
@@ -29,19 +29,12 @@ class GetRequests(CachedFunction):
 
     def execute(self):
         # return [_format_message(msg) for msg in EXAMPLE_DATA]
-        from hubs.app import app
-        url = "%s/raw" % app.config["DATAGREPPER_URI"]
-        args = {
+        raw_messages = datagrepper_get({
             "topic": self.TOPIC,
             "delta": str(86400 * 10),  # 10 days
-        }
-        log.info("Getting request data from Datagrepper")
-        response = requests.get(url, args)
-        if response.status_code != 200:
-            log.warning("Could not get requests from Datagrepper: %s",
-                        response.text)
-            return []
-        raw_messages = response.json()["raw_messages"]
+        })
+        if raw_messages is None:
+            return None
         return [_format_message(msg) for msg in raw_messages]
 
     def should_invalidate(self, message):
diff --git a/hubs/widgets/halp/views.py b/hubs/widgets/halp/views.py
index 584fb53..a00ad6e 100644
--- a/hubs/widgets/halp/views.py
+++ b/hubs/widgets/halp/views.py
@@ -40,8 +40,9 @@ class DataView(WidgetView):
     def get_context(self, instance, *args, **kwargs):
         get_requests = GetRequests(instance)
         data = {"requests": []}
+        requests = get_requests() or []
         hubs_filter = flask.request.args.getlist("hubs")
-        for req in get_requests():
+        for req in requests:
             req["hubs"] = find_hubs_for_msg(req)
             if not req["hubs"]:
                 continue
@@ -93,7 +94,8 @@ class SearchView(WidgetView):
 
         # filter the data
         data = {"requests": []}
-        for req in get_requests():
+        requests = get_requests() or []
+        for req in requests:
             req["hubs"] = find_hubs_for_msg(req)
             if not req["hubs"]:
                 continue
@@ -127,8 +129,9 @@ class RequestersView(WidgetView):
 
     def get_context(self, instance, *args, **kwargs):
         get_requests = GetRequests(instance)
+        requests = get_requests() or []
         results = set([
-            req["author"]["name"] for req in get_requests()
+            req["author"]["name"] for req in requests
             ])
         query = flask.request.args.get("q")
         if query:

From 14abe1437b42ae8ef39ceac188d663e5ec6a07c8 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard <aurelien@bompard.org>
Date: Aug 09 2017 15:17:24 +0000
Subject: [PATCH 2/2] Fix documentation


---

diff --git a/docs/api/utils.rst b/docs/api/utils.rst
index e178be4..9154b45 100644
--- a/docs/api/utils.rst
+++ b/docs/api/utils.rst
@@ -19,3 +19,9 @@ Packages database utils
 .. automodule:: hubs.utils.pkgdb
    :members:
 
+Datagrepper utils
+-----------------
+
+.. automodule:: hubs.utils.datagrepper
+   :members:
+
diff --git a/hubs/utils/datagrepper.py b/hubs/utils/datagrepper.py
index 42aa8ec..0a266aa 100644
--- a/hubs/utils/datagrepper.py
+++ b/hubs/utils/datagrepper.py
@@ -4,8 +4,6 @@ import logging
 
 import requests
 
-from hubs.app import app
-
 log = logging.getLogger(__name__)
 
 
@@ -16,7 +14,12 @@ def datagrepper_get(args):
 
     Parameters:
         args (dict): elements of the query string to Datagrepper.
+
+    Returns:
+        list of dictionaries: The accumulated Fedmsg messages in the
+        ``raw_messages`` lists over every page.
     """
+    from hubs.app import app  # Avoid circular import
     url = "%s/raw" % app.config["DATAGREPPER_URI"]
     result = []
     page = 1