From 986891624f25b2566efd003487723a4dd889b8f4 Mon Sep 17 00:00:00 2001 From: Zbigniew Jędrzejewski-Szmek Date: Apr 03 2016 05:48:43 +0000 Subject: [PATCH 1/3] Fix typo in error handling --- diff --git a/spectool b/spectool index 565550c..2a08978 100755 --- a/spectool +++ b/spectool @@ -81,7 +81,7 @@ def run(*popenargs, timeout=None, **kwargs): def error(message, exception=None): print(message, file=sys.stderr) if exception is not None: - print(e, file=sys.stderr) + print(exception, file=sys.stderr) sys.exit(1) class Spec(object): From 463a7c59ae1bd70152804d56a15a8906a22985dc Mon Sep 17 00:00:00 2001 From: Zbigniew Jędrzejewski-Szmek Date: Apr 03 2016 05:48:43 +0000 Subject: [PATCH 2/3] Extract asset list generation from listfiles --- diff --git a/spectool b/spectool index 2a08978..9f67563 100755 --- a/spectool +++ b/spectool @@ -358,28 +358,33 @@ def download_files(spec, opts, selected): print(dir) - -def listfiles(spec, opts, selected): +def generate_asset_list(spec, opts, selected): if opts.allsources or opts.all: - for source, num in sorted(zip(spec.sources, spec.sourcenums), key=operator.itemgetter(1)): - print("Source{}: {}".format(num, source)) + for num, source in sorted(zip(spec.sourcenums, spec.sources)): + yield 'Source', num, source if opts.allpatches or opts.all: - for patch, num in sorted(zip(spec.patches, spec.patchnums), key=operator.itemgetter(1)): - print("Patch{}: {}".format(num, patch)) + for num, patch in sorted(zip(spec.patchnums, spec.patches)): + yield 'Patch', num, patch for source in opts.sourcelist: if source in spec.sourcenums: - print("Source{}: {}".format(source, spec.sources[source])) + yield 'Source', source, spec.sources[source] else: - print('No source item {}.'.format(source)) + yield 'Error', source, 'No source item {}'.format(source) for patch in opts.patchlist: if patch in spec.patchnums: - print("Patch{}: {}".format(patch, spec.patches[patch])) + yield 'Patch', patch, spec.patches[patch] else: - print('No patch item {}.'.format(patch)) + yield 'Error', patch, 'No patch item {}'.format(patch) +def listfiles(spec, opts, selected): + for typ, num, asset in generate_asset_list(spec, opts, selected): + if typ == 'Error': + print(asset) + else: + print('{}{}: {}'.format(typ, num, asset)) def show_parsed_data(spec, opts): print("Parsed these tags:") From e88aac87ec55ab0d7b2de1a50b032dc9233701a8 Mon Sep 17 00:00:00 2001 From: Zbigniew Jędrzejewski-Szmek Date: Apr 03 2016 05:48:44 +0000 Subject: [PATCH 3/3] Implement -g --dryrun, --force options work. Tested with: Source0: ftp://ftp.ntua.gr/pub/linux/fedora/linux/releases/23/Everything/armhfp/os/Packages/j/junit-4.12-3.fc23.noarch.rpm Source1: ftp://ftp.muug.mb.ca/mirror/fedora/linux/releases/23/Server/armhfp/os/Packages/j/junit-4.12-3.fc23.noarch.rpm#/different-name Source2: https://github.com/systemd/python-systemd/archive/v231.zip Source3: http://github.com/systemd/python-systemd/archive/v231.zip#/v231-over-http.zip Source4: http://example.com/v231.zip#/bad-url.zip (last one fails obviously) --- diff --git a/spectool b/spectool index 9f67563..e05c807 100755 --- a/spectool +++ b/spectool @@ -4,7 +4,10 @@ import operator import glob import re import sys +import shutil +import os.path from subprocess import CalledProcessError, PIPE, Popen, TimeoutExpired +from urllib import request # Python conversion of spectool. # Spectool has two functions: @@ -22,6 +25,8 @@ CURLRC = '/etc/rpmdevrools/curlrc' PROTOCOLS = ['ftp', 'http', 'https'] dbprint = None +USER_AGENT = 'spectool/' + VERSION + # Sure wish I had Python 3.5's subprocess.run(), so here's a hacked one. class CompletedProcess(object): @@ -134,7 +139,6 @@ class Spec(object): self.patches.append(val) self.patchnums.append(int(m.group('pnum') or 0)) - class SelectionError(Exception): pass @@ -308,56 +312,6 @@ def get_download_location(spec, opts): return expand_sourcedir_macro(spec) -def is_downloadable(url): - """Check that string is a valid URL of a protocol which CURL can handle.""" - return False - - -def download_files(spec, opts, selected): - """ - Fetch the sources. - - Here's the relevant perl code: - if (retrievable ($url)) { - my $path = File::Spec->catfile($where, $url =~ m|([^/]+)$|); - print "Getting $url to $path\n"; - if (-e $path) { - if ($force) { - if (! unlink $path) { - warn("Could not unlink $path, skipping download: $!\n"); - return 1; - } - } else { - warn("$path already exists, skipping download\n"); - return 0; - } - } - # Note: -k/--insecure is intentionally not here; add it to - # $CURLRC if you want it. - my @cmd = (qw (curl --fail --remote-time --location - --output), $path, - '--user-agent', "spectool/$VERSION"); - push(@cmd, '--config', $CURLRC) if (-e $CURLRC); - push(@cmd, $url); - print "--> @cmd\n" if ($verbose > 1); - if (! $dryrun) { - system @cmd; - return $? == -1 ? 127 : $? >> 8; - } else { - print "dry run: @cmd\n"; - } - } else { - warn "Couldn't fetch $url: missing/unsupported URL\n" if ($verbose); - } - return 0; - """ - dir = get_download_location(spec, opts) - # urls = [] - - # Iterate over sources - - print(dir) - def generate_asset_list(spec, opts, selected): if opts.allsources or opts.all: for num, source in sorted(zip(spec.sourcenums, spec.sources)): @@ -386,6 +340,39 @@ def listfiles(spec, opts, selected): else: print('{}{}: {}'.format(typ, num, asset)) + +def is_downloadable(url): + """Check that string is a valid URL of a protocol which we can handle.""" + return url.split('://')[0] in {'http', 'https', 'ftp'} + +def path_download_name(url): + return url.split('/')[-1] + +def download_file(url, dest): + req = request.Request(url, headers={'User-Agent': USER_AGENT}) + with request.urlopen(req) as inp: + with open(dest, 'wb') as out: + shutil.copyfileobj(inp, out) + +def download_files(spec, opts, selected): + """ + Fetch the sources. + """ + dir = get_download_location(spec, opts) + for typ, num, asset in generate_asset_list(spec, opts, selected): + if typ == 'Error': + raise IndexError(asset) + if not is_downloadable(asset): + print('{}{}: {} cannot be downloaded'.format(typ, num, asset)) + else: + dest = os.path.join(dir, path_download_name(asset)) + if not opts.force and os.path.exists(dest): + print('{}{}: {} already exists'.format(typ, num, dest)) + else: + print('{}{}: {} → {}'.format(typ, num, asset, dest)) + if not opts.dryrun: + download_file(asset, dest) + def show_parsed_data(spec, opts): print("Parsed these tags:") print("-> Name: {}".format(spec.name))