From ff85c383a3e5a45aa4f1eb084b4c7399afce1f96 Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 14 2017 21:45:05 +0000 Subject: [PATCH 1/3] Introduce --autoemails in github importer Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/commands/github.py b/pagure_importer/commands/github.py index f0fd058..3adf463 100644 --- a/pagure_importer/commands/github.py +++ b/pagure_importer/commands/github.py @@ -30,8 +30,10 @@ import pagure_importer.utils.git as gitutils @click.option('--is_fork', is_flag=True, default=False) @click.option('--nosearch', is_flag=True, help="Do not go through the list of commits to find email ids") +@click.option('--autoemails', is_flag=True, + help="Generate fake emails of the users whose email is not known") def github(username, project, nopush, pagure_project, - status, gencsv, namespace, is_fork, nosearch): + status, gencsv, namespace, is_fork, nosearch, autoemails): ''' For imports from github ''' password = click.prompt("Github Password", hide_input=True) @@ -65,7 +67,8 @@ def github(username, project, nopush, pagure_project, repo = github_importer.github.get_repo( github_importer.github_project_name) - github_importer.import_issues(repo, status=status) + github_importer.import_issues( + repo, status=status, autoemails=autoemails) # update the local git repo new_repo = gitutils.update_git( diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index e42ecf4..16d59e9 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -159,6 +159,12 @@ def gh_get_issue_users(github_username, github_password, github_project_name): return +def _gen_email(username): + ''' Generate a fake email id ''' + + return username + '@' + 'fake.pgimport.com' + + def gh_assemble_users(): ''' It uses the files: issue_commentors.json and contributors.json Assembles and creates a file: assembled_commentors.csv @@ -184,6 +190,7 @@ def gh_assemble_users(): d = {'name': i, 'fullname': None, 'emails': None} names.append(d) + with open('assembled_users.csv', 'w') as ac: field_names = ['name', 'fullname', 'emails'] writer = csv.DictWriter(ac, fieldnames=field_names) @@ -193,7 +200,7 @@ def gh_assemble_users(): writer.writerow(name) -def gh_get_user_email(name): +def gh_get_user_email(name, autoemails=False): ''' Will return the issue commentor email as given in the assembled_users.csv file ''' @@ -213,12 +220,15 @@ def gh_get_user_email(name): ('emails', row['emails'])))) for i in data: - if i.get('name', None) == name: + if i['name'] == name: if i['emails']: return str(i['emails']) + + if autoemails: + return _gen_email(username=i['name']) else: raise EmailNotFound('You need to fill out all the emails of' - ' the issue commentors') + ' the issue commentors or use --autoemails') def get_pagure_namespace(repo_folder, repo_name): diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index 98528ca..f8e68f2 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -130,7 +130,7 @@ class GithubImporter(object): return whole_body, pagure_attachments - def import_issues(self, repo, status='all'): + def import_issues(self, repo, status='all', autoemails=False): ''' Imports the issues on github for the given project ''' repo_issues = repo.get_issues(state=status) @@ -221,7 +221,11 @@ class GithubImporter(object): name=comment_user.login, fullname=comment_user.name, emails=[comment_user.email] if comment_user.email - else [gh_get_user_email(comment_user.login)]) + else [gh_get_user_email( + comment_user.login, + autoemails=autoemails + )] + ) # Object to represent comment on an issue pagure_issue_comment = models.IssueComment( From ff579a42881b53411a00b57dc37db12c8c61460a Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 14 2017 21:45:05 +0000 Subject: [PATCH 2/3] Pass on autoemails options in assignee and comment user Signed-off-by: Vivek Anand --- diff --git a/pagure_importer/utils/importer_github.py b/pagure_importer/utils/importer_github.py index f8e68f2..839b327 100644 --- a/pagure_importer/utils/importer_github.py +++ b/pagure_importer/utils/importer_github.py @@ -41,7 +41,7 @@ class GithubImporter(object): if not self.nopush: shutil.rmtree(self.clone_repo_location) - def get_issue_assignee(self, github_issue): + def get_issue_assignee(self, github_issue, autoemails): ''' From the github issue object, return the assignee of the issue if any ''' @@ -50,7 +50,10 @@ class GithubImporter(object): assignee = models.User( name=github_issue.assignee.login, fullname=github_issue.assignee.name, - emails=[gh_get_user_email(github_issue.assignee.login)] + emails=[gh_get_user_email( + github_issue.assignee.login, + autoemails=autoemails + )] ) if assignee is not None: @@ -158,7 +161,8 @@ class GithubImporter(object): pagure_issue_created_at = github_issue.created_at.strftime('%s') # Get the assignee of the issue - pagure_issue_assignee = self.get_issue_assignee(github_issue) + pagure_issue_assignee = self.get_issue_assignee( + github_issue, autoemails=autoemails) if github_issue.labels: pagure_issue_tags = [i.name for i in github_issue.labels] @@ -178,7 +182,11 @@ class GithubImporter(object): name=github_issue.user.login, fullname=github_issue.user.name, emails=[github_issue.user.email] if github_issue.user.email - else [gh_get_user_email(github_issue.user.login)]) + else [gh_get_user_email( + github_issue.user.login, + autoemails=autoemails + )] + ) pagure_issue = models.Issue( id=None, From 2c32975261ef772aef92dcd182511c25c6091b8f Mon Sep 17 00:00:00 2001 From: Vivek Anand Date: Mar 14 2017 21:48:56 +0000 Subject: [PATCH 3/3] Update readme for --autoemails Signed-off-by: Vivek Anand --- diff --git a/README.md b/README.md index 72a2434..f13709f 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,8 @@ To add some new close status just edit the config file as follow. Where ```Foo`` This will give out ```assembled_users.csv``` with all the users whose email ids must be known and should be filled before going to the next step. + Since, to fill the csv file can be very tiring and one wants to get the + import done, read about ```pgimport github --autoemails``` in the next point. 3) Once the csv is filled: @@ -171,6 +173,17 @@ To add some new close status just edit the config file as follow. Where ```Foo`` The issues will be imported to /tmp/foobar.git repository. + Since, there can be a lot of fields in the csv file. There is another option + which can make life a little easier: + + $ pgimport github --autoemails + + This option has some consequences. If a user is not found corresponding + to the ```name``` field in csv, pagure creates a user with that ```name``` and the + corresponding email given. In case the email is not valid, a user is created in + pagure's database that will never be used again. The user will never get any + notification if he/she is supposed to get. + 3) The push command can be used to push a clone pagure ticket repo back to pagure. $ pgimport push foobar.git