From 393bc1d9e15433101f75a877e3fd1fdf97230349 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 1/7] kojid: use mergerepo_c for all merge modes Fixes: https://pagure.io/koji/issue/2183 --- diff --git a/builder/Makefile b/builder/Makefile index 9d16298..dccac78 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -1,3 +1,5 @@ +PYVER := $(shell $(PYTHON) -c 'import sys; print("%.3s" %(sys.version))') + BINFILES = kojid LIBEXECFILES = mergerepos SYSTEMDSYSTEMUNITDIR = $(shell pkg-config systemd --variable=systemdsystemunitdir) @@ -21,8 +23,10 @@ _install: mkdir -p $(DESTDIR)/usr/sbin install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin - mkdir -p $(DESTDIR)/usr/libexec/kojid - install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid + @if [ "$(PYVER)" -gt 2 ] ; then \ + mkdir -p $(DESTDIR)/usr/libexec/kojid; \ + install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid; \ + fi mkdir -p $(DESTDIR)/etc/mock/koji diff --git a/builder/kojid b/builder/kojid index afe6636..9a1a44a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -37,6 +37,7 @@ import shutil import signal import smtplib import socket +import subprocess import sys import time import traceback @@ -5508,6 +5509,19 @@ class CreaterepoTask(BaseTaskHandler): raise koji.GenericError('failed to create repo: %s' % parseStatus(status, ' '.join(cmd))) + def _get_mergerepo_c_version(self): + cmd = ['/usr/bin/mergerepo_c', '--version'] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out, _ = proc.communicate() + status = proc.wait() + if status != 0: + return None + out = out.decode().strip() + #Version: 0.15.11 (Features: DeltaRPM LegacyWeakdeps ) + version = re.match('Version: (\d+).(\d+).(\d+).*', out).groups() + version = [int(x) for x in version] + return tuple(version) + def merge_repos(self, external_repos, arch, groupdata): # group repos by merge type repos_by_mode = {} @@ -5535,26 +5549,38 @@ class CreaterepoTask(BaseTaskHandler): ext_url = ext_url.replace('$arch', arch) repos.append(ext_url) + if self.options.use_createrepo_c or six.PY3: + version = self._get_mergerepo_c_version() + if version is None: + raise koji.GenericError("mergerepo_c is not installed") + # construct command if merge_mode == 'simple': - # currently only supported by our own mergerepos script - # (we need it to write pkgorigins) - cmd = ['/usr/libexec/kojid/mergerepos', - '--mode', 'simple', - '--tempdir', self.workdir] + if self.options.use_createrepo_c or six.PY3: + version = self._get_mergerepo_c_version() + if version < (0, 15, 1): + raise koji.GenericError("mergerepo_c is not installed or has low " + "version: %s (0.15.1 - required)" % version) + cmd = ['/usr/bin/mergerepo_c', '--koji', '--simple'] + else: + cmd = ['/usr/libexec/kojid/mergerepos', + '--mode', 'simple', + '--tempdir', self.workdir] elif merge_mode == 'bare': # "bare" merge mode for repos with modular metadata # forces use of mergerepo_c cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all'] - elif self.options.use_createrepo_c: + elif self.options.use_createrepo_c or six.PY3: cmd = ['/usr/bin/mergerepo_c', '--koji'] else: - cmd = ['/usr/libexec/kojid/mergerepos'] - cmd.extend(['--tempdir', self.workdir]) + cmd = ['/usr/libexec/kojid/mergerepos', '--tempdir', self.workdir] + if merge_mode != 'bare': blocklist = self.repodir + '/blocklist' cmd.extend(['-b', blocklist]) cmd.extend(['-a', arch, '-o', self.outdir]) + if cmd[0].endswith('mergerepo_c') and version > (0, 15, 10): + cmd.append('--arch-expand') if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) for repo in repos: diff --git a/koji.spec b/koji.spec index 77d6df2..16adcb2 100644 --- a/koji.spec +++ b/koji.spec @@ -563,8 +563,10 @@ rm -rf $RPM_BUILD_ROOT %files builder %{_sbindir}/kojid +%if 0%{py2_support} > 1 %dir %{_libexecdir}/kojid %{_libexecdir}/kojid/mergerepos +%endif %if %{use_systemd} %{_unitdir}/kojid.service %else From 3a6bfec59e52a543e80518294c892e8c4d764c37 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 2/7] fix required version --- diff --git a/builder/Makefile b/builder/Makefile index dccac78..b137335 100644 --- a/builder/Makefile +++ b/builder/Makefile @@ -1,4 +1,4 @@ -PYVER := $(shell $(PYTHON) -c 'import sys; print("%.3s" %(sys.version))') +PYVER := $(shell $(PYTHON) -c 'import sys; print("%.1s" %(sys.version))') BINFILES = kojid LIBEXECFILES = mergerepos @@ -23,7 +23,7 @@ _install: mkdir -p $(DESTDIR)/usr/sbin install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin - @if [ "$(PYVER)" -gt 2 ] ; then \ + @if [ "$(PYVER)" -lt 3 ] ; then \ mkdir -p $(DESTDIR)/usr/libexec/kojid; \ install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid; \ fi diff --git a/builder/kojid b/builder/kojid index 9a1a44a..5d063e7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5558,9 +5558,9 @@ class CreaterepoTask(BaseTaskHandler): if merge_mode == 'simple': if self.options.use_createrepo_c or six.PY3: version = self._get_mergerepo_c_version() - if version < (0, 15, 1): - raise koji.GenericError("mergerepo_c is not installed or has low " - "version: %s (0.15.1 - required)" % version) + if version < (0, 13, 0): + raise koji.GenericError("mergerepo_c is not installed or has low version: " + "%s (0.13.0 - required for --simple)" % version) cmd = ['/usr/bin/mergerepo_c', '--koji', '--simple'] else: cmd = ['/usr/libexec/kojid/mergerepos', From 72bc9ed3eb4ae75c51d9d8c47614c28683678858 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 3/7] make sure we fall back to mergerepos on python2 also catch more error cases and adjust logging --- diff --git a/builder/kojid b/builder/kojid index 5d063e7..cfb48b4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5515,10 +5515,15 @@ class CreaterepoTask(BaseTaskHandler): out, _ = proc.communicate() status = proc.wait() if status != 0: + self.logger.warning("Unable to detect mergerepo_c version") return None out = out.decode().strip() - #Version: 0.15.11 (Features: DeltaRPM LegacyWeakdeps ) - version = re.match('Version: (\d+).(\d+).(\d+).*', out).groups() + # Expects output like: "Version: 0.15.11 (Features: DeltaRPM LegacyWeakdeps )" + m = re.match('Version: (\d+).(\d+).(\d+).*', out) + if not m: + self.logger.warning("Unable to parse mergerepo_c version") + return None + version = m.groups() version = [int(x) for x in version] return tuple(version) @@ -5549,19 +5554,18 @@ class CreaterepoTask(BaseTaskHandler): ext_url = ext_url.replace('$arch', arch) repos.append(ext_url) + mergerepo_c_version = None if self.options.use_createrepo_c or six.PY3: - version = self._get_mergerepo_c_version() - if version is None: - raise koji.GenericError("mergerepo_c is not installed") + mergerepo_c_version = self._get_mergerepo_c_version() # construct command if merge_mode == 'simple': - if self.options.use_createrepo_c or six.PY3: - version = self._get_mergerepo_c_version() - if version < (0, 13, 0): - raise koji.GenericError("mergerepo_c is not installed or has low version: " - "%s (0.13.0 - required for --simple)" % version) + if mergerepo_c_version and mergerepo_c_version >= (0, 13, 0): cmd = ['/usr/bin/mergerepo_c', '--koji', '--simple'] + elif six.PY3: + # koji's mergerepos script only works on python2 + raise koji.GenericError("mergerepo_c is not installed or has low version: " + "%s (0.13.0 needed for --simple)" % mergerepo_c_version) else: cmd = ['/usr/libexec/kojid/mergerepos', '--mode', 'simple', @@ -5579,7 +5583,7 @@ class CreaterepoTask(BaseTaskHandler): blocklist = self.repodir + '/blocklist' cmd.extend(['-b', blocklist]) cmd.extend(['-a', arch, '-o', self.outdir]) - if cmd[0].endswith('mergerepo_c') and version > (0, 15, 10): + if cmd[0].endswith('mergerepo_c') and mergerepo_c_version >= (0, 15, 11): cmd.append('--arch-expand') if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) From 921bc30573a83170296c9e3113bec43d8749d903 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 4/7] safe convertion from tuple to str for error msg --- diff --git a/builder/kojid b/builder/kojid index cfb48b4..27f53fb 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5565,7 +5565,8 @@ class CreaterepoTask(BaseTaskHandler): elif six.PY3: # koji's mergerepos script only works on python2 raise koji.GenericError("mergerepo_c is not installed or has low version: " - "%s (0.13.0 needed for --simple)" % mergerepo_c_version) + "%s (0.13.0 needed for --simple)" % + ".".join([str(d) for d in mergerepo_c_version or [None]])) else: cmd = ['/usr/libexec/kojid/mergerepos', '--mode', 'simple', From 022ee9dbce3d7581cf5568d584a198153342fe48 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 5/7] do not break things when mergerepo_c version is not detectable --- diff --git a/builder/kojid b/builder/kojid index 27f53fb..b419468 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5584,7 +5584,8 @@ class CreaterepoTask(BaseTaskHandler): blocklist = self.repodir + '/blocklist' cmd.extend(['-b', blocklist]) cmd.extend(['-a', arch, '-o', self.outdir]) - if cmd[0].endswith('mergerepo_c') and mergerepo_c_version >= (0, 15, 11): + if cmd[0].endswith('mergerepo_c') and mergerepo_c_version \ + and mergerepo_c_version >= (0, 15, 11): cmd.append('--arch-expand') if os.path.isfile(groupdata): cmd.extend(['-g', groupdata]) From cccd7f06a32d11832628d2d3551a9091deddeb24 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 6/7] flake8: W605 invalid escape sequence '\d' --- diff --git a/builder/kojid b/builder/kojid index b419468..65da243 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5519,7 +5519,7 @@ class CreaterepoTask(BaseTaskHandler): return None out = out.decode().strip() # Expects output like: "Version: 0.15.11 (Features: DeltaRPM LegacyWeakdeps )" - m = re.match('Version: (\d+).(\d+).(\d+).*', out) + m = re.match(r'Version: (\d+).(\d+).(\d+).*', out) if not m: self.logger.warning("Unable to parse mergerepo_c version") return None From a02762bc7fe3e2860c241b5f8ec77e2d6050a011 Mon Sep 17 00:00:00 2001 From: Yu Ming Zhu Date: Jul 16 2020 15:17:44 +0000 Subject: [PATCH 7/7] depress Exception when checking mergerepo_c version --- diff --git a/builder/kojid b/builder/kojid index 65da243..78ad92a 100755 --- a/builder/kojid +++ b/builder/kojid @@ -5511,10 +5511,14 @@ class CreaterepoTask(BaseTaskHandler): def _get_mergerepo_c_version(self): cmd = ['/usr/bin/mergerepo_c', '--version'] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) - out, _ = proc.communicate() - status = proc.wait() - if status != 0: + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) + out, _ = proc.communicate() + status = proc.wait() + if status != 0: + self.logger.warning("Unable to detect mergerepo_c version") + return None + except Exception: self.logger.warning("Unable to detect mergerepo_c version") return None out = out.decode().strip()