From af9395c7f3d89cc0ab505ce4d186f22bc7962325 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 04 2016 22:08:09 +0000 Subject: [PATCH 1/5] Fix urlparse and encoding for python3 Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 22042ab..3132467 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -3,8 +3,8 @@ import os import sys import json import click -import urlparse import pygit2 +from urllib.parse import urlparse from configparser import ConfigParser from github import Github from github.GithubException import TwoFactorException @@ -221,7 +221,7 @@ def get_pagure_namespace(repo_folder, repo_name): repo_path = os.path.join(repo_folder, repo_name) repo = pygit2.Repository(repo_path) remote_url = repo.remotes['origin'].url - remote_path = urlparse.urlparse(remote_url).path + remote_path = urlparse(remote_url).path remote_path = remote_path.replace('.git', '') namespace_list = remote_path.split('/')[2:] namespace = '/'.join(namespace_list) diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index c337325..0321010 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -14,9 +14,8 @@ import werkzeug def get_secure_filename(attachment, filename): ''' Hashes the file name, same as pagure ''' - attach = str(attachment) - filename = '%s-%s' % (hashlib.sha256(attach).hexdigest(), - werkzeug.secure_filename(unicode(filename))) + filename = '%s-%s' % (hashlib.sha256(attachment).hexdigest(), + werkzeug.secure_filename(str(filename))) return filename @@ -78,7 +77,7 @@ def update_git(obj, newpath, new_repo): attach_path = os.path.join(newpath, 'files', filename) with open(attach_path, 'w') as stream: stream.write(str(attachments[key])) - index.add('files/'+filename) + index.add('files/' + filename) # Write down what changed with open(file_path, 'w') as stream: From d931daef65819f8692ff46fef8eb682a01864b6e Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 04 2016 22:09:39 +0000 Subject: [PATCH 2/5] Fixed image decoding/encoding from trac to pagure Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index 3132467..dac9e3f 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -4,6 +4,7 @@ import sys import json import click import pygit2 +import re from urllib.parse import urlparse from configparser import ConfigParser from github import Github @@ -226,3 +227,12 @@ def get_pagure_namespace(repo_folder, repo_name): namespace_list = remote_path.split('/')[2:] namespace = '/'.join(namespace_list) return namespace + + +def is_image(filename): + ''' True is filename extension is .jpg, .png, .gif, .bmp or .jpeg else False''' + + if re.match('\w+\.(jpg|png|gif|bmp|jpeg)', filename) is not None: + return True + else: + return False diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index 0321010..ef5240b 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -10,7 +10,7 @@ import pygit2 import json import hashlib import werkzeug - +from pagure_importer.utils import is_image def get_secure_filename(attachment, filename): ''' Hashes the file name, same as pagure ''' @@ -75,8 +75,12 @@ def update_git(obj, newpath, new_repo): for key in attachments.keys(): filename = get_secure_filename(attachments[key], key) attach_path = os.path.join(newpath, 'files', filename) - with open(attach_path, 'w') as stream: - stream.write(str(attachments[key])) + if is_image(filename): + with open(attach_path, 'wb') as stream: + stream.write(attachments[key]) + else: + with open(attach_path, 'w') as stream: + stream.write(str(attachments[key])) index.add('files/' + filename) # Write down what changed diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index 8302083..f040973 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -1,12 +1,12 @@ -import re import sys +import re import time -import base64 import click import requests - +from base64 import b64decode from datetime import datetime -from pagure_importer.utils import get_pagure_namespace, get_close_status +from pagure_importer.utils import ( + get_pagure_namespace, get_close_status, is_image) from pagure_importer.utils.git import ( clone_repo, get_secure_filename, push_delete_repo, update_git) from pagure_importer.utils.models import User, Issue, IssueComment @@ -141,8 +141,12 @@ class TracImporter(): 'ticket.getAttachment', ticket_id, filename) if attachment_resp: - content = attachment_resp['__jsonclass__'][1].replace('\n', '') - pagure_attachment[filename] = base64.b64decode(content) + if is_image(filename): + content = b64decode(attachment_resp['__jsonclass__'][1]) + pagure_attachment[filename] = content + else: + content = b64decode(attachment_resp['__jsonclass__'][1].replace('\n', '')) + pagure_attachment[filename] = content.decode() pagure_custom_fields = self.get_custom_fields_of_ticket(trac_ticket) pagure_issue_title = trac_ticket['summary'] @@ -164,13 +168,13 @@ class TracImporter(): pagure_issue_user = User( name=trac_ticket['reporter'], fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter']+'@fedoraproject.org']) + emails=[trac_ticket['reporter'] + '@fedoraproject.org']) else: pagure_issue_assignee = User(name='', fullname='', emails=[]) pagure_issue_user = User( name=trac_ticket['reporter'], fullname=trac_ticket['reporter'], - emails=[trac_ticket['reporter']+'@fedoraproject.org']) + emails=[trac_ticket['reporter'] + '@fedoraproject.org']) # The milestone of the issue pagure_milestone = None From 8cf2ebbe8199a2eedce244c7675ed1f662b137d6 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 04 2016 22:09:39 +0000 Subject: [PATCH 3/5] Fixing missing space in prompt Signed-off-by: Clement Verna --- diff --git a/pagure_importer/commands/push.py b/pagure_importer/commands/push.py index bc2c3d3..413dabd 100644 --- a/pagure_importer/commands/push.py +++ b/pagure_importer/commands/push.py @@ -8,7 +8,7 @@ from pagure_importer.app import app, REPO_PATH @click.argument('repo_name') def push(repo_name): if click.confirm('Before executing this command, you must have' - '"Pagure Tickets" enabled from pagure project\'s settings. Continue?'): + ' "Pagure Tickets" enabled from pagure project\'s settings. Continue?'): repo = os.path.join(REPO_PATH, repo_name) os.chdir(repo) cmd = ['git', 'push', 'origin', 'master'] From 65d7066a959d273989f5efb23e3d77c2d75cecbc Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 04 2016 22:09:39 +0000 Subject: [PATCH 4/5] Nice display of images in pagure Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index ef5240b..938423f 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -75,12 +75,12 @@ def update_git(obj, newpath, new_repo): for key in attachments.keys(): filename = get_secure_filename(attachments[key], key) attach_path = os.path.join(newpath, 'files', filename) - if is_image(filename): + if is_image(key): with open(attach_path, 'wb') as stream: stream.write(attachments[key]) else: with open(attach_path, 'w') as stream: - stream.write(str(attachments[key])) + stream.write(attachments[key].decode()) index.add('files/' + filename) # Write down what changed diff --git a/pagure_importer/utils/importer_trac.py b/pagure_importer/utils/importer_trac.py index f040973..af3e6ef 100644 --- a/pagure_importer/utils/importer_trac.py +++ b/pagure_importer/utils/importer_trac.py @@ -103,8 +103,12 @@ class TracImporter(): filename = get_secure_filename( pagure_issue.attachment[attach_name], attach_name) url = '/%s/issue/raw/files/%s' % (project, filename) - comments[key].comment += ('\n[%s](%s)' % - (attach_name, url)) + if is_image(attach_name): + comments[key].comment += ('\n[![%s](%s)](%s)' % + (attach_name, url, url)) + else: + comments[key].comment += ('\n[%s](%s)' % + (attach_name, url)) pagure_issue.comments.append(comments[key].to_json()) # update the local git repo new_repo = update_git(pagure_issue, newpath, new_repo) @@ -145,8 +149,9 @@ class TracImporter(): content = b64decode(attachment_resp['__jsonclass__'][1]) pagure_attachment[filename] = content else: - content = b64decode(attachment_resp['__jsonclass__'][1].replace('\n', '')) - pagure_attachment[filename] = content.decode() + content = b64decode( + attachment_resp['__jsonclass__'][1].replace('\n', '')) + pagure_attachment[filename] = content pagure_custom_fields = self.get_custom_fields_of_ticket(trac_ticket) pagure_issue_title = trac_ticket['summary'] From e83fd058f0eb82ef16fa4102bb92792aad54a55f Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Dec 05 2016 22:05:38 +0000 Subject: [PATCH 5/5] Fixed the way we write the data to the attachment file. If it can be utf-8 decoded then write as text else keep it as binary Signed-off-by: Clement Verna --- diff --git a/pagure_importer/utils/__init__.py b/pagure_importer/utils/__init__.py index dac9e3f..13fa426 100644 --- a/pagure_importer/utils/__init__.py +++ b/pagure_importer/utils/__init__.py @@ -232,7 +232,8 @@ def get_pagure_namespace(repo_folder, repo_name): def is_image(filename): ''' True is filename extension is .jpg, .png, .gif, .bmp or .jpeg else False''' - if re.match('\w+\.(jpg|png|gif|bmp|jpeg)', filename) is not None: + if re.search('\.(jpg|png|gif|bmp|jpeg|JPG|PNG|GIF|BMP|JPEG)', + filename) is not None: return True else: return False diff --git a/pagure_importer/utils/git.py b/pagure_importer/utils/git.py index 938423f..6b8ea13 100644 --- a/pagure_importer/utils/git.py +++ b/pagure_importer/utils/git.py @@ -75,12 +75,15 @@ def update_git(obj, newpath, new_repo): for key in attachments.keys(): filename = get_secure_filename(attachments[key], key) attach_path = os.path.join(newpath, 'files', filename) - if is_image(key): - with open(attach_path, 'wb') as stream: - stream.write(attachments[key]) - else: + # Try decoding Bytes to UTF-8 + try: with open(attach_path, 'w') as stream: stream.write(attachments[key].decode()) + # If it fails write the data as binary + except UnicodeDecodeError: + with open(attach_path, 'wb') as stream: + stream.write(attachments[key]) + index.add('files/' + filename) # Write down what changed