From 5770c132bbdc0cbbc2a11d88b566b3e04653ef75 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 09 2017 20:23:21 +0000
Subject: [PATCH 1/2] Add Support for multi-threaded import of fedorahosted issues


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index eb1d20c..c6c3bfe 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -3,6 +3,9 @@ import re
 import time
 import click
 import requests
+from threading import Thread
+from random import randint
+from queue import Queue
 from base64 import b64decode
 from datetime import datetime
 from pagure_importer.utils import (
@@ -34,20 +37,19 @@ class TracImporter(Importer):
         self.offset = offset
         self.somebody = User(name='somebody', fullname='somebody',
                              emails=['some@body.com'])
-        self.reqid = 0
         self.custom_fields = self.get_custom_fields()
+        self.queue = Queue()
 
     def request(self, method, *args):
         ''' Common method for querying trac '''
-
-        self.reqid += 1
+        reqid = randint(0,100)
         req = {'params': args,
                'method': method,
-               'id': self.reqid}
+               'id': reqid}
         resp = requests.post(self.url, json=req,
                              auth=(self.username, self.password))
         resp = resp.json()
-        if resp['id'] != self.reqid:
+        if resp['id'] != reqid:
             click.echo('ERROR: Invalid response for request! ' +
                        'ID does not match')
             sys.exit(1)
@@ -91,8 +93,19 @@ class TracImporter(Importer):
         '''
 
         tickets_id = self.request('ticket.query', trac_query)
+        for i in range(10):
+            t = Thread(target=self.import_thread, args=(self.queue, repo_name, tickets_id))
+            t.setDaemon(True)
+            t.start()
 
         for ticket_id in tickets_id:
+            self.queue.put(ticket_id)
+
+        self.queue.join()
+
+    def import_thread(self, queue, repo_name, tickets_id):
+        while True:
+            ticket_id = queue.get()
             pagure_issue = self.create_issue(ticket_id)
             pagure_issue.comments = []
             pagure_issue_comments = self.request('ticket.changeLog', ticket_id)
@@ -117,6 +130,7 @@ class TracImporter(Importer):
             click.echo('Updated ' + repo_name + ' with issue :' +
                        str(ticket_id) + '/' + str(tickets_id[-1]))
             issue_to_json(pagure_issue, self.clone_repo_location)
+            self.queue.task_done()
 
     def get_custom_fields_of_ticket(self, trac_ticket):
         ''' Given the trac ticket, it will return all the

From 3dbc95a765ea49b3cadd478476348d11163637c7 Mon Sep 17 00:00:00 2001
From: Clement Verna <cverna@tutanota.com>
Date: Jan 09 2017 20:46:30 +0000
Subject: [PATCH 2/2] Make the number of thread a parameter


Signed-off-by: Clement Verna <cverna@tutanota.com>

---

diff --git a/README.md b/README.md
index 3ac953a..c86d876 100644
--- a/README.md
+++ b/README.md
@@ -84,9 +84,10 @@ To add some new close status just edit the config file as follow. Where ```Foo``
         --private By default make all issues private.
         --username TEXT FAS username
         --password TEXT FAS password
-        --offset INTEGER Number of issue in pagure before import
+        --offset INTEGER Number of issue in pagure before import (Default = 0)
         --help  Show this message and exit.
         --nopush Do not push the result of pagure-importer back
+        --threads Number of thread used during the issue import (Default = 5)
 
 
     $ pgimport fedorahosted https://fedorahosted.org/foobar --tags
@@ -114,6 +115,10 @@ To add some new close status just edit the config file as follow. Where ```Foo``
    representation of the issues to import. This can be used to process the issues using the json files
    before running the import.
    Default location of the cloned repository is under /tmp/clone-foobar.git
+   
+    $ pgimport fedorahosted https://fedorahosted.org/foobar --threads 50
+    
+   This command will run the importer using 50 threads to import the issues in parallel.
 
 3) The push command can be used to push a clone pagure ticket repo back to pagure.
 
diff --git a/pagure_importer/commands/fedorahosted.py b/pagure_importer/commands/fedorahosted.py
index b3b1986..1c19c9d 100644
--- a/pagure_importer/commands/fedorahosted.py
+++ b/pagure_importer/commands/fedorahosted.py
@@ -20,8 +20,10 @@ import pagure_importer.utils.git as gitutils
               help='Number of issue in pagure before import')
 @click.option('--nopush', is_flag=True,
               help="Do not push the result of pagure-importer back")
+@click.option('--threads', default=5,
+              help="Number of thread used during the issue import")
 def fedorahosted(
-        project_url, tags, private, username, password, offset, nopush):
+        project_url, tags, private, username, password, offset, nopush, threads):
     fasclient = FASclient(username, password,
                           'https://admin.fedoraproject.org/accounts')
     project_url += '/login/jsonrpc'
@@ -41,7 +43,8 @@ def fedorahosted(
                                         fasclient=fasclient,
                                         tags=tags,
                                         private=private,
-                                        nopush=nopush) as trac_importer:
+                                        nopush=nopush,
+                                        threads=threads) as trac_importer:
 
             trac_importer.import_issues(project)
             # update the local git repo
diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py
index c6c3bfe..a5f1a0d 100644
--- a/pagure_importer/utils/importer_trac.py
+++ b/pagure_importer/utils/importer_trac.py
@@ -27,7 +27,7 @@ class TracImporter(Importer):
     ''' Pagure importer for trac instance '''
 
     def __init__(self, project_url, username, password, offset, repo_name,
-                 repo_folder, nopush, fasclient=None, tags=False, private=False):
+                 repo_folder, nopush, threads, fasclient=None, tags=False, private=False):
         ''' Instantiate a TracImporter object '''
         Importer.__init__(self, username, password, repo_name, repo_folder, nopush)
         self.url = project_url
@@ -35,6 +35,7 @@ class TracImporter(Importer):
         self.tags = tags
         self.private = private
         self.offset = offset
+        self.threads = threads
         self.somebody = User(name='somebody', fullname='somebody',
                              emails=['some@body.com'])
         self.custom_fields = self.get_custom_fields()
@@ -42,7 +43,7 @@ class TracImporter(Importer):
 
     def request(self, method, *args):
         ''' Common method for querying trac '''
-        reqid = randint(0,100)
+        reqid = randint(0, self.threads)
         req = {'params': args,
                'method': method,
                'id': reqid}
@@ -93,7 +94,7 @@ class TracImporter(Importer):
         '''
 
         tickets_id = self.request('ticket.query', trac_query)
-        for i in range(10):
+        for i in range(self.threads):
             t = Thread(target=self.import_thread, args=(self.queue, repo_name, tickets_id))
             t.setDaemon(True)
             t.start()

