From b2c37233f1f5e5c69ee5134b6258634f7c2e4cdb Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Feb 07 2024 14:24:53 +0000 Subject: [PATCH 1/4] Track uploads in BuildRoot and log mock config Related: https://pagure.io/koji/issue/3875 --- diff --git a/builder/kojid b/builder/kojid index b4536dd..291dba7 100755 --- a/builder/kojid +++ b/builder/kojid @@ -199,6 +199,7 @@ class BuildRoot(object): self.logger = logging.getLogger("koji.build.buildroot") self.session = session self.options = options + self.logs = set() if len(args) + len(kwargs) == 1: # manage an existing mock buildroot self._load(*args, **kwargs) @@ -346,6 +347,8 @@ class BuildRoot(object): with koji._open_text_file(configfile, 'wt') as fo: fo.write(output) + self.single_log(configfile, name='mock_config.log') + def get_repo_dir(self): pathinfo = koji.PathInfo(topdir='') return pathinfo.repo(self.repoid, self.tag_name) @@ -468,7 +471,6 @@ class BuildRoot(object): pid = os.fork() if pid: resultdir = self.resultdir() - uploadpath = self.getUploadPath() logs = {} ts_offsets = {} @@ -550,7 +552,7 @@ class BuildRoot(object): with koji._open_text_file(fpath, 'at') as ts_file: ts_file.write('%.0f %i\n' % (time.time(), position)) ts_offsets[fname] = position - incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger) + self.incremental_log(fname, fd) # clean up and return exit status of command for (fname, (fd, inode, size, fpath)) in logs.items(): @@ -559,7 +561,7 @@ class BuildRoot(object): if fname.endswith('-ts.log'): # finish upload of ts.log as they could've been missed in # last iteration - incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger) + self.incremental_log(fname, fd) fd.close() return status[1] @@ -591,6 +593,17 @@ class BuildRoot(object): the hub.""" return koji.pathinfo.taskrelpath(self.task_id) + def incremental_log(self, fname, fd): + ret = incremental_upload(self.session, fname, fd, self.getUploadPath(), logger=self.logger) + self.logs.add(fname) + return ret + + def single_log(self, fname, name=None): + if name is None: + name = os.path.basename(fname) + self.session.uploadWrapper(fname, self.getUploadPath(), name=name) + self.logs.add(name) + def init(self): rv = self.mock(['--init']) @@ -1550,12 +1563,13 @@ class BuildArchTask(BaseBuildTask): resultdir = broot.resultdir() rpm_files = [] srpm_files = [] - log_files = [] + log_files = list(broot.logs) unexpected = [] for f in os.listdir(resultdir): # files here should have one of two extensions: .log and .rpm - if f[-4:] == ".log": - log_files.append(f) + if f[-4:] in (".log"): + pass + # should already be in log_files elif f[-8:] == ".src.rpm": srpm_files.append(f) elif f[-4:] == ".rpm": @@ -1563,12 +1577,6 @@ class BuildArchTask(BaseBuildTask): else: unexpected.append(f) - if broot.workdir: - mocklog = 'mock_output.log' - f = os.path.join(broot.workdir, mocklog) - if os.path.exists(f): - log_files.append(os.path.basename(f)) - # for noarch rpms compute rpmdiff hash rpmdiff_hash = {self.id: {}} for rpmf in rpm_files: @@ -1580,6 +1588,7 @@ class BuildArchTask(BaseBuildTask): log_name = 'noarch_rpmdiff.json' noarch_hash_path = os.path.join(broot.workdir, log_name) koji.dump_json(noarch_hash_path, rpmdiff_hash, indent=2, sort_keys=True) + self.uploadFile(noarch_hash_path) log_files.append(log_name) self.logger.debug("rpms: %r" % rpm_files) @@ -1613,8 +1622,6 @@ class BuildArchTask(BaseBuildTask): else: ret['srpms'] = [] ret['logs'] = ["%s/%s" % (uploadpath, f) for f in log_files] - if rpmdiff_hash[self.id]: - self.uploadFile(noarch_hash_path) ret['brootid'] = broot.id @@ -1948,16 +1955,8 @@ class BuildMavenTask(BaseBuildTask): self.uploadFile(os.path.join(outputdir, relpath, filename), relPath=relpath) - # Should only find log files in the mock result directory. - # Don't upload these log files, they've already been streamed - # the hub. - for filename in os.listdir(buildroot.resultdir()): - root, ext = os.path.splitext(filename) - if ext == '.log': - filepath = os.path.join(buildroot.resultdir(), filename) - if os.path.isfile(filepath) and os.stat(filepath).st_size > 0: - # only files with content get uploaded to the hub - logs.append(filename) + # Also include the logs already upload by BuildRoot + logs.extend(broot.logs) buildroot.expire() @@ -2274,7 +2273,7 @@ class WrapperRPMTask(BaseBuildTask): srpm = None rpms = [] specfile_name = os.path.basename(specfile) - logs = ['checkout.log', specfile_name] + logs = ['checkout.log', specfile_name] + list(buildroot.logs) for filename in os.listdir(resultdir): if filename.endswith('.src.rpm'): @@ -2288,7 +2287,8 @@ class WrapperRPMTask(BaseBuildTask): elif filename.endswith('.rpm'): rpms.append(filename) elif filename.endswith('.log'): - logs.append(filename) + pass + # already included in buildroot.logs else: if self.new_build_id: self.session.host.failBuild(self.id, self.new_build_id) @@ -5129,14 +5129,13 @@ class RebuildSRPM(BaseBuildTask): self.uploadFile(srpm) brootid = broot.id - log_files = glob.glob('%s/*.log' % broot.resultdir()) + log_files = list(broot.logs) broot.expire() return { 'srpm': "%s/%s" % (uploadpath, srpm_name), - 'logs': ["%s/%s" % (uploadpath, os.path.basename(f)) - for f in log_files], + 'logs': ["%s/%s" % (uploadpath, f) for f in log_files], 'brootid': brootid, 'source': { 'source': os.path.basename(srpm), @@ -5299,13 +5298,12 @@ class BuildSRPMFromSCMTask(BaseBuildTask): self.uploadFile(srpm) brootid = broot.id - log_files = glob.glob('%s/*.log' % broot.resultdir()) + log_files = list(broot.logs) broot.expire() return {'srpm': "%s/%s" % (uploadpath, srpm_name), - 'logs': ["%s/%s" % (uploadpath, os.path.basename(f)) - for f in log_files], + 'logs': ["%s/%s" % (uploadpath, f) for f in log_files], 'brootid': brootid, 'source': source, } From 99d4752c2ce427a99c7bb397160c503850fa3704 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Feb 07 2024 14:24:55 +0000 Subject: [PATCH 2/4] rename var --- diff --git a/builder/kojid b/builder/kojid index 291dba7..b47cbda 100755 --- a/builder/kojid +++ b/builder/kojid @@ -598,10 +598,10 @@ class BuildRoot(object): self.logs.add(fname) return ret - def single_log(self, fname, name=None): + def single_log(self, localfile, name=None): if name is None: - name = os.path.basename(fname) - self.session.uploadWrapper(fname, self.getUploadPath(), name=name) + name = os.path.basename(localfile) + self.session.uploadWrapper(localfile, self.getUploadPath(), name=name) self.logs.add(name) def init(self): From ffa93628d58660c028ae0dba3eb52a3219999566 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Feb 07 2024 14:24:55 +0000 Subject: [PATCH 3/4] also use in appliance/livecd/liveimage --- diff --git a/builder/kojid b/builder/kojid index b47cbda..040b776 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3425,13 +3425,13 @@ class ApplianceTask(ImageTask): self.logger.debug('output: %s' % results) if len(results) == 0: raise koji.ApplianceError("Could not find image build results!") + logs = ['appliance.log', os.path.basename(ksfile), os.path.basename(kskoji)] + logs.extend(broot.logs) imgdata = { 'arch': arch, 'rootdev': self.getRootDevice(), 'task_id': self.id, - 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'appliance.log', os.path.basename(ksfile), - os.path.basename(kskoji)], + 'logs': logs, 'name': name, 'version': version, 'release': release @@ -3597,13 +3597,13 @@ class LiveCDTask(ImageTask): self.uploadFile(manifest) self.uploadFile(isosrc, remoteName=isoname) + logs = ['livecd.log', os.path.basename(ksfile), os.path.basename(kskoji)] + logs.extend(broot.logs) imgdata = {'arch': arch, 'files': [isoname], 'rootdev': None, 'task_id': self.id, - 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livecd.log', os.path.basename(ksfile), - os.path.basename(kskoji)], + 'logs': logs, 'name': name, 'version': version, 'release': release @@ -3852,13 +3852,13 @@ class LiveMediaTask(ImageTask): self.logger.debug('uploading image: %s' % isosrc) self.uploadFile(isosrc, remoteName=isoname) + logs = ['livemedia-out.log', os.path.basename(ksfile), os.path.basename(kskoji)] + logs.extend(broot.logs) imgdata = {'arch': arch, 'files': [isoname], 'rootdev': None, 'task_id': self.id, - 'logs': ['build.log', 'mock_output.log', 'root.log', 'state.log', - 'livemedia-out.log', os.path.basename(ksfile), - os.path.basename(kskoji)], + 'logs': logs, 'name': name, 'version': version, 'release': release From 4792b334191c493e9fa5f144a9bb467def8f553e Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Feb 28 2024 02:20:12 +0000 Subject: [PATCH 4/4] typo in BuildMavenTask --- diff --git a/builder/kojid b/builder/kojid index 040b776..fa36132 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1956,7 +1956,7 @@ class BuildMavenTask(BaseBuildTask): relPath=relpath) # Also include the logs already upload by BuildRoot - logs.extend(broot.logs) + logs.extend(buildroot.logs) buildroot.expire()