From 42c6dae6b24f324ca18d1465018f8ce505c827ab Mon Sep 17 00:00:00 2001 From: Christos Triantafyllidis Date: Jun 21 2016 21:30:09 +0000 Subject: Initial DEB support (importing and listing) --- diff --git a/cli/koji b/cli/koji index 0abe244..0d6a9a2 100755 --- a/cli/koji +++ b/cli/koji @@ -62,8 +62,13 @@ import urlgrabber.progress as progress import xmlrpclib import yum.comps import optparse +from hashlib import sha1 #for import-comps handler (currently disabled) #from rhpl.comps import Comps +try: + from debian.debfile import DebFile +except ImportError: + pass # fix OptionParser for python 2.3 (optparse verion 1.4.1+) # code taken from optparse version 1.5a2 @@ -1651,6 +1656,123 @@ def handle_import(options, session, args): do_import(path, data) +def handle_import_deb(options, session, args): + "[admin] Import externally built DEBs into the database" + if 'debian.debfile' not in sys.modules: + raise koji.GenericError, "The 'debian' python module is missing" + usage = _("usage: %prog import_deb [options] package [package...]") + usage += _("\n(Specify the --help global option for a list of other help options)") + parser = OptionParser(usage=usage) + parser.add_option("--link", action="store_true", help=_("Attempt to hardlink instead of uploading")) + parser.add_option("--test", action="store_true", help=_("Don't actually import")) + parser.add_option("--create-build", action="store_true", help=_("Auto-create builds as needed")) + (options, args) = parser.parse_args(args) + if len(args) < 1: + parser.error(_("At least one package must be specified")) + assert False + activate_session(session) + to_import = {} + for path in args: + debfile_control = DebFile(path).control.debcontrol() + debinfo = {} + debinfo['name'] = str(debfile_control['Package']) + debinfo['source'] = str(debfile_control['Source']) + debinfo['version'], debinfo['release'] = str(debfile_control['Version']).split('-') + debinfo['arch'] = str(debfile_control['Architecture']) + with open(path, 'rb') as deb_file: + debinfo['payloadhash'] = sha1(deb_file.read()).hexdigest() + + nvr = "%(source)s-%(version)s-%(release)s" % debinfo + to_import.setdefault(nvr,[]).append((path,debinfo)) + builds_missing = False + nvrs = to_import.keys() + nvrs.sort() + for nvr in nvrs: + to_import[nvr].sort() + binfo = session.getBuild(nvr) + if not binfo: + print _("Missing build: %s") % nvr + builds_missing = True + if builds_missing and not options.create_build: + print _("Aborting import") + return + + #local function to help us out below + def do_import(path, data): + rinfo = dict([(k,data[k]) for k in ('name','version','release','arch')]) + prev = session.getDEB(rinfo) + print prev + if prev and not prev.get('external_repo_id', 0): + if prev['payloadhash'] == data['payloadhash']: + print _("DEB already imported: %s") % path + else: + print _("WARNING: sha1 mismatch for %s") % path + print _("Skipping import") + return + if options.test: + print _("Test mode -- skipping import for %s") % path + return + serverdir = _unique_path('cli-import') + if options.link: + linked_upload(path, serverdir) + else: + print _("uploading %s...") % path, + sys.stdout.flush() + session.uploadWrapper(path, serverdir) + print _("done") + sys.stdout.flush() + print _("importing %s...") % path, + sys.stdout.flush() + try: + session.importDEB(serverdir, os.path.basename(path)) + except koji.GenericError, e: + print _("\nError importing: %s" % str(e).splitlines()[-1]) + sys.stdout.flush() + else: + print _("done") + sys.stdout.flush() + + for nvr in nvrs: + # check for existing build + need_build = True + binfo = session.getBuild(nvr) + if binfo: + b_state = koji.BUILD_STATES[binfo['state']] + if b_state == 'COMPLETE': + need_build = False + elif b_state in ['FAILED', 'CANCELED']: + if not options.create_build: + print _("Build %s state is %s. Skipping import") % (nvr, b_state) + continue + else: + print _("Build %s exists with state=%s. Skipping import") % (nvr, b_state) + continue + + if need_build: + if not options.create_build: + if binfo: + # should have caught this earlier, but just in case... + b_state = koji.BUILD_STATES[binfo['state']] + print _("Build %s state is %s. Skipping import") % (nvr, b_state) + continue + else: + print _("No such build: %s (use --create-build option to add it)") % nvr + continue + else: + # let's make a new build + b_data = koji.parse_NVR(nvr) + b_data['epoch'] = None + if options.test: + print _("Test mode -- would have created empty build: %s") % nvr + else: + print _("Creating empty build: %s") % nvr + session.createEmptyBuild(**b_data) + binfo = session.getBuild(nvr) + + for path, data in to_import[nvr]: + do_import(path, data) + + def handle_import_cg(options, session, args): "[admin] Import external builds with rich metadata" usage = _("usage: %prog import-cg [options] metadata_file files_dir") @@ -2697,6 +2819,7 @@ def anon_handle_list_tagged(options, session, args): parser = OptionParser(usage=usage) parser.add_option("--arch", help=_("List rpms for this arch")) parser.add_option("--rpms", action="store_true", help=_("Show rpms instead of builds")) + parser.add_option("--debs", action="store_true", help=_("Show debs instead of builds")) parser.add_option("--inherit", action="store_true", help=_("Follow inheritance")) parser.add_option("--latest", action="store_true", help=_("Only show the latest builds/rpms")) parser.add_option("--latest-n", type='int', metavar="N", help=_("Only show the latest N builds/rpms")) @@ -2762,6 +2885,19 @@ def anon_handle_list_tagged(options, session, args): fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" if options.sigs: fmt = "%(sigkey)s " + fmt + elif options.debs: + debs, builds = session.listTaggedDEBS(tag, **opts) + data = debs + if options.paths: + build_idx = dict([(b['id'],b) for b in builds]) + for rinfo in data: + build = build_idx[rinfo['build_id']] + builddir = pathinfo.build(build) + rinfo['path'] = os.path.join(builddir, pathinfo.deb(rinfo)) + fmt = "%(path)s" + data = [x for x in data if x.has_key('path')] + else: + fmt = "%(name)s_%(version)s-%(release)s_%(arch)s" else: data = session.listTagged(tag, **opts) if options.paths: diff --git a/docs/schema-deb.sql b/docs/schema-deb.sql new file mode 100644 index 0000000..c2329c7 --- /dev/null +++ b/docs/schema-deb.sql @@ -0,0 +1,26 @@ + +BEGIN; + +-- debinfo tracks individual debs +-- buildroot_id can be NULL (for externally built packages) +-- we demand that N-V-R_A be unique we don't store filename +-- because filename should be N-V-R_A.deb +CREATE TABLE debinfo ( + id SERIAL NOT NULL PRIMARY KEY, + build_id INTEGER REFERENCES build (id), + buildroot_id INTEGER REFERENCES buildroot (id), + name TEXT NOT NULL, + version TEXT NOT NULL, + release TEXT NOT NULL, + arch VARCHAR(16) NOT NULL, + source TEXT NOT NULL, + external_repo_id INTEGER NOT NULL REFERENCES external_repo(id), + size BIGINT NOT NULL, + payloadhash TEXT NOT NULL, + metadata_only BOOLEAN NOT NULL DEFAULT FALSE, + extra TEXT, + CONSTRAINT debinfo_unique_nvra UNIQUE (name,version,release,arch,external_repo_id) +) WITHOUT OIDS; +CREATE INDEX debinfo_build ON debinfo(build_id); + +COMMIT; diff --git a/hub/kojihub.py b/hub/kojihub.py index 2ea1689..1bea933 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -56,6 +56,10 @@ import types import xmlrpclib import zipfile from koji.context import context +try: + from debian.debfile import DebFile +except ImportError: + pass logger = logging.getLogger('koji.hub') @@ -1214,6 +1218,91 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne return builds +def readTaggedDEBS(tag, package=None, arch=None, event=None,inherit=False,latest=True,owner=None,type=None): + """Returns a list of dems for specified tag + + set inherit=True to follow inheritance + set event to query at a time in the past + set latest=False to get all tagged DEBS (not just from the latest builds) + set latest=N to get only the N latest tagged DEBs + + If type is not None, restrict the list to debs from builds of the given type. Currently the + supported types are 'maven' and 'win'. + """ + taglist = [tag] + if inherit: + #XXX really should cache this - it gets called several places + # (however, it is fairly quick) + taglist += [link['parent_id'] for link in readFullInheritance(tag, event)] + + builds = readTaggedBuilds(tag, event=event, inherit=inherit, latest=latest, package=package, owner=owner, type=type) + #index builds + build_idx = dict([(b['build_id'],b) for b in builds]) + + #the following query is run for each tag in the inheritance + fields = [('debinfo.name', 'name'), + ('debinfo.version', 'version'), + ('debinfo.release', 'release'), + ('debinfo.arch', 'arch'), + ('debinfo.id', 'id'), + ('debinfo.size', 'size'), + ('debinfo.payloadhash', 'payloadhash'), + ('debinfo.buildroot_id', 'buildroot_id'), + ('debinfo.build_id', 'build_id')] + tables = ['debinfo'] + joins = ['tag_listing ON debinfo.build_id = tag_listing.build_id'] + clauses = [eventCondition(event), 'tag_id=%(tagid)s'] + data = {} #tagid added later + if package: + joins.append('build ON debinfo.build_id = build.id') + joins.append('package ON package.id = build.pkg_id') + clauses.append('package.name = %(package)s') + data['package'] = package + if arch: + data['arch'] = arch + if isinstance(arch, basestring): + clauses.append('debinfo.arch = %(arch)s') + elif isinstance(arch, (list, tuple)): + clauses.append('debinfo.arch IN %(arch)s') + else: + raise koji.GenericError, 'invalid arch option: %s' % arch + + fields, aliases = zip(*fields) + query = QueryProcessor(tables=tables, joins=joins, clauses=clauses, + columns=fields, aliases=aliases, values=data, transform=_fix_deb_row) + + # unique constraints ensure that each of these queries will not report + # duplicate debinfo entries, BUT since we make the query multiple times, + # we can get duplicates if a package is multiply tagged. + debs = [] + tags_seen = {} + def _iter_debs(): + for tagid in taglist: + if tags_seen.has_key(tagid): + #certain inheritance trees can (legitimately) have the same tag + #appear more than once (perhaps once with a package filter and once + #without). The hard part of that was already done by readTaggedBuilds. + #We only need consider each tag once. Note how we use build_idx below. + #(Without this, we could report the same deb twice) + continue + else: + tags_seen[tagid] = 1 + query.values['tagid'] = tagid + for debinfo in query.iterate(): + #note: we're checking against the build list because + # it has been filtered by the package list. The tag + # tools should endeavor to keep tag_listing sane w.r.t. + # the package list, but if there is disagreement the package + # list should take priority + build = build_idx.get(debinfo['build_id'],None) + if build is None: + continue + elif build['tag_id'] != tagid: + #wrong tag + continue + yield debinfo + return [_iter_debs(), builds] + def readTaggedRPMS(tag, package=None, arch=None, event=None,inherit=False,latest=True,rpmsigs=False,owner=None,type=None): """Returns a list of rpms for specified tag @@ -3370,7 +3459,102 @@ def _fix_rpm_row(row): #alias for now, may change in the future _fix_archive_row = _fix_rpm_row +_fix_deb_row = _fix_rpm_row + +def get_deb(debinfo, strict=False, multi=False): + """Get information about the specified DEB + + debinfo may be any one of the following: + - a int ID + - a string N-V-R.A + - a string N-V-R.A@location + - a map containing 'name', 'version', 'release', and 'arch' + (and optionally 'location') + + If specified, location should match the name of an external repo + + A map will be returned, with the following keys: + - id + - name + - version + - release + - arch + - playloadhash + - size + - build_id + - buildroot_id + - external_repo_id + - external_repo_name + - metadata_only + - extra + If there is no DEB with the given ID, None is returned, unless strict + is True in which case an exception is raised + + If more than one DEB matches, and multi is True, then a list of results is + returned. If multi is False, a single match is returned (an internal one if + possible). + """ + fields = ( + ('debinfo.id', 'id'), + ('build_id', 'build_id'), + ('buildroot_id', 'buildroot_id'), + ('debinfo.name', 'name'), + ('version', 'version'), + ('release', 'release'), + ('arch', 'arch'), + ('external_repo_id', 'external_repo_id'), + ('external_repo.name', 'external_repo_name'), + ('debinfo.payloadhash', 'payloadhash'), + ('size', 'size'), + ('metadata_only', 'metadata_only'), + ('extra', 'extra'), + ) + # we can look up by id or NVRA + data = None + if isinstance(debinfo,(int,long)): + data = {'id': debinfo} + elif isinstance(debinfo,str): + data = koji.parse_NVRA(debinfo) + elif isinstance(debinfo,dict): + data = debinfo.copy() + else: + raise koji.GenericError, "Invalid argument: %r" % debinfo + clauses = [] + if data.has_key('id'): + clauses.append("debinfo.id=%(id)s") + else: + clauses.append("""debinfo.name=%(name)s AND version=%(version)s + AND release=%(release)s AND arch=%(arch)s""") + retry = False + if data.has_key('location'): + data['external_repo_id'] = get_external_repo_id(data['location'], strict=True) + clauses.append("""external_repo_id = %(external_repo_id)i""") + elif not multi: + #try to match internal first, otherwise first matching external + retry = True #if no internal match + orig_clauses = list(clauses) #copy + clauses.append("""external_repo_id = 0""") + + joins = ['external_repo ON debinfo.external_repo_id = external_repo.id'] + + query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], + tables=['debinfo'], joins=joins, clauses=clauses, + values=data, transform=_fix_deb_row) + if multi: + return query.execute() + ret = query.executeOne() + if ret: + return ret + if retry: + #at this point we have just an NVRA with no internal match. Open it up to externals + query.clauses = orig_clauses + ret = query.executeOne() + if not ret: + if strict: + raise koji.GenericError, "No such deb: %r" % data + return None + return ret def get_rpm(rpminfo, strict=False, multi=False): """Get information about the specified RPM @@ -3471,6 +3655,71 @@ def get_rpm(rpminfo, strict=False, multi=False): return None return ret +def list_debs(buildID=None, buildrootID=None, imageID=None, hostID=None, arches=None, queryOpts=None): + """List DEBS. If buildID, imageID and/or buildrootID are specified, + restrict the list of RPMs to only those RPMs that are part of that + build, or were built in that buildroot. If componentBuildrootID is specified, + restrict the list to only those RPMs that will get pulled into that buildroot + when it is used to build another package. A list of maps is returned, each map + containing the following keys: + - id + - name + - version + - release + - nvr (synthesized for sorting purposes) + - arch + - playloadhash + - size + - build_id + - buildroot_id + - external_repo_id + - external_repo_name + - metadata_only + - extra + If no build has the given ID, or the build generated no RPMs, + an empty list is returned.""" + fields = [('debinfo.id', 'id'), ('debinfo.name', 'name'), ('debinfo.version', 'version'), + ('debinfo.release', 'release'), + ("debinfo.name || '-' || debinfo.version || '-' || debinfo.release", 'nvr'), + ('debinfo.arch', 'arch'), + ('debinfo.payloadhash', 'payloadhash') + ('debinfo.size', 'size'), + ('debinfo.build_id', 'build_id'), ('debinfo.buildroot_id', 'buildroot_id'), + ('debinfo.external_repo_id', 'external_repo_id'), + ('external_repo.name', 'external_repo_name'), + ('debinfo.metadata_only', 'metadata_only'), + ('debinfo.extra', 'extra'), + ] + joins = ['external_repo ON debinfo.external_repo_id = external_repo.id'] + clauses = [] + + if buildID != None: + clauses.append('debinfo.build_id = %(buildID)i') + if buildrootID != None: + clauses.append('debinfo.buildroot_id = %(buildrootID)i') + + # image specific constraints + if imageID != None: + clauses.append('image_listing.image_id = %(imageID)i') + joins.append('image_listing ON debinfo.id = image_listing.deb_id') + + if hostID != None: + joins.append('standard_buildroot ON debinfo.buildroot_id = standard_buildroot.buildroot_id') + clauses.append('standard_buildroot.host_id = %(hostID)i') + if arches != None: + if isinstance(arches, list) or isinstance(arches, tuple): + clauses.append('debinfo.arch IN %(arches)s') + elif isinstance(arches, str): + clauses.append('debinfo.arch = %(arches)s') + else: + raise koji.GenericError, 'invalid type for "arches" parameter: %s' % type(arches) + + fields, aliases = zip(*fields) + query = QueryProcessor(columns=fields, aliases=aliases, + tables=['debinfo'], joins=joins, clauses=clauses, + values=locals(), transform=_fix_deb_row, opts=queryOpts) + data = query.execute() + return data def list_rpms(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None): """List RPMS. If buildID, imageID and/or buildrootID are specified, @@ -4590,6 +4839,73 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) return binfo +def import_deb(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): + """Import a single deb into the database + + Designed to be called from import_build_deb. + """ + if 'debian.debfile' not in sys.modules: + raise koji.GenericError, "The 'debian' python module is missing on the hub" + + if not os.path.exists(fn): + raise koji.GenericError, "no such file: %s" % fn + + #read deb info + debfile_control = DebFile(fn).control.debcontrol() + debinfo = {} + debinfo['name'] = str(debfile_control['Package']) + debinfo['source'] = str(debfile_control['Source']) + debinfo['version'], debinfo['release'] = str(debfile_control['Version']).split('-') + debinfo['arch'] = str(debfile_control['Architecture']) + with open(fn, 'rb') as deb_file: + debinfo['payloadhash'] = hashlib.sha1(deb_file.read()).hexdigest() + + #sanity check basename + basename = os.path.basename(fn) + expected = "%(name)s_%(version)s-%(release)s_%(arch)s.deb" % debinfo + if basename != expected: + raise koji.GenericError, "bad filename: %s (expected %s)" % (basename,expected) + + if buildinfo is None: + #figure it out from NVR + buildinfo = get_build("%(source)s-%(version)s-%(release)s" % debinfo) + if buildinfo is None: + #XXX - handle case where package is not a source rpm + # and we still need to create a new build + raise koji.GenericError, 'No matching build' + state = koji.BUILD_STATES[buildinfo['state']] + if state in ('FAILED', 'CANCELED', 'DELETED'): + nvr = "%(name)s-%(version)s-%(release)s" % buildinfo + raise koji.GenericError, "Build is %s: %s" % (state, nvr) + + #add debinfo entry + debinfo['id'] = _singleValue("""SELECT nextval('debinfo_id_seq')""") + debinfo['build_id'] = buildinfo['id'] + debinfo['size'] = os.path.getsize(fn) + debinfo['buildroot_id'] = brootid + debinfo['external_repo_id'] = 0 + + # handle cg extra info + if fileinfo is not None: + extra = fileinfo.get('extra') + if extra is not None: + rpminfo['extra'] = json.dumps(extra) + + koji.plugin.run_callbacks('preImport', type='deb', deb=debinfo, build=buildinfo, + filepath=fn) + + data = debinfo.copy() + insert = InsertProcessor('debinfo', data=data) + insert.execute() + + koji.plugin.run_callbacks('postImport', type='deb', deb=debinfo, build=buildinfo, + filepath=fn) + + #extra fields for return + debinfo['build'] = buildinfo + debinfo['brootid'] = brootid + return debinfo + def import_rpm(fn, buildinfo=None, brootid=None, wrapper=False, fileinfo=None): """Import a single rpm into the database @@ -5144,6 +5460,14 @@ def import_build_log(fn, buildinfo, subdir=None): os.rename(fn,final_path) os.symlink(final_path,fn) +def import_deb_file(fn,buildinfo,debinfo): + """Move the rpm file into the proper place + + Generally this is done after the db import + """ + final_path = "%s/%s" % (koji.pathinfo.build(buildinfo),koji.pathinfo.deb(debinfo)) + _import_archive_file(fn, os.path.dirname(final_path)) + def import_rpm_file(fn,buildinfo,rpminfo): """Move the rpm file into the proper place @@ -8557,6 +8881,21 @@ class RootExports(object): build = get_build(build_id, strict=True) new_image_build(build) + def importDEB(self, path, basename): + """Import an DEB into the database. + + The file must be uploaded first. + """ + context.session.assertPerm('admin') + uploadpath = koji.pathinfo.work() + fn = "%s/%s/%s" %(uploadpath,path,basename) + if not os.path.exists(fn): + raise koji.GenericError, "No such file: %s" % fn + debinfo = import_deb(fn) + import_deb_file(fn,debinfo['build'],debinfo) + for tag in list_tags(build=debinfo['build_id']): + set_tag_update(tag['id'], 'IMPORT') + def importRPM(self, path, basename): """Import an RPM into the database. @@ -8952,6 +9291,13 @@ class RootExports(object): results = [build for build in results if build['package_name'].lower().startswith(prefix)] return results + def listTaggedDEBS(self,tag,event=None,inherit=False,latest=False,package=None,arch=None,owner=None,type=None): + """List debs and builds within tag""" + if not isinstance(tag,int): + #lookup tag id + tag = get_tag_id(tag,strict=True) + return readTaggedDEBS(tag,event=event,inherit=inherit,latest=latest,package=package,arch=arch,owner=owner,type=type) + def listTaggedRPMS(self,tag,event=None,inherit=False,latest=False,package=None,arch=None,rpmsigs=False,owner=None,type=None): """List rpms and builds within tag""" if not isinstance(tag,int): @@ -9202,6 +9548,30 @@ class RootExports(object): mapping[int(key)] = mapping[key] return readFullInheritance(tag,event,reverse,stops,jumps) + listDEBs = staticmethod(list_debs) + + def listBuildDEBs(self,build): + """Get information about all the DEBs generated by the build with the given + ID. A list of maps is returned, each map containing the following keys: + + - id + - name + - version + - release + - arch + - epoch + - payloadhash + - size + - buildtime + - build_id + - buildroot_id + + If no build has the given ID, or the build generated no DEBs, an empty list is returned.""" + if not isinstance(build, int): + #lookup build id + build = self.findBuildID(build, strict=True) + return self.listDEBs(buildID=build) + listRPMs = staticmethod(list_rpms) def listBuildRPMs(self,build): @@ -9228,6 +9598,43 @@ class RootExports(object): getRPM = staticmethod(get_rpm) + getDEB = staticmethod(get_deb) + + def getDEBDeps(self, debID, depType=None, queryOpts=None): + """Return dependency information about the DEB with the given ID. + If depType is specified, restrict results to dependencies of the given type. + Otherwise, return all dependency information. A list of maps will be returned, + each with the following keys: + - name + - type + + If there is no DEB with the given ID, or the DEB has no dependency information, + an empty list will be returned. + """ + if queryOpts is None: + queryOpts = {} + deb_info = get_deb(debID) + if not deb_info or not deb_info['build_id']: + return _applyQueryOpts([], queryOpts) + build_info = get_build(deb_info['build_id']) + deb_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.deb(deb_info)) + if not os.path.exists(deb_path): + return _applyQueryOpts([], queryOpts) + + results = [] + + for dep_name in ['Depends','Recommends','Suggests', 'Pre-Depends', 'Build-Depends', 'Build-Depends-Indep']: + if depType is None or depType.lower() == dep_name.lower(): + debcontrol = DebFile(deb_path).debcontrol() + if dep_name in debcontrol: + for name in debcontrol[dep_name].split(", "): + if queryOpts.get('asList'): + results.append([name, dep_name]) + else: + results.append({'name': name, 'type': dep_name}) + + return _applyQueryOpts(results, queryOpts) + def getRPMDeps(self, rpmID, depType=None, queryOpts=None): """Return dependency information about the RPM with the given ID. If depType is specified, restrict results to dependencies of the given type. @@ -9269,6 +9676,52 @@ class RootExports(object): return _applyQueryOpts(results, queryOpts) + def listDEBFiles(self, debID, queryOpts=None): + """List files associated with the DEB with the given ID. A list of maps + will be returned, each with the following keys: + - name + - digest + - md5 (alias for digest) + - digest_algo + - size + - flags + + If there is no DEB with the given ID, or that DEB contains no files, + an empty list will be returned.""" + + if 'debian.debfile' not in sys.modules: + raise koji.GenericError, "The 'debian' python module is missing on the hub" + + if queryOpts is None: + queryOpts = {} + deb_info = get_deb(debID) + if not deb_info or not deb_info['build_id']: + return _applyQueryOpts([], queryOpts) + build_info = get_build(deb_info['build_id']) + deb_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.deb(deb_info)) + if not os.path.exists(deb_path): + return _applyQueryOpts([], queryOpts) + + results = [] + digest_algo = 'md5' # DEB always uses MD5 + flags = 0 # DEB files don't have flags + + debfile = DebFile(deb_path) + + for filename in debfile.md5sums().keys(): + try: + data_member = debfile.data.tgz().getmember(filename) + except KeyError: + data_member = debfile.data.tgz().getmember('./' + filename) + if queryOpts.get('asList'): + results.append([filename, debfile.md5sums()[filename], data_member.size, flags, digest_algo, data_member.uname, data_member.gname, data_member.mtime, data_member.mode]) + else: + results.append({'name': filename, 'digest': debfile.md5sums()[filename], 'digest_algo': digest_algo, + 'md5': debfile.md5sums()[filename], 'size': data_member.size, 'flags': flags, + 'user': data_member.uname, 'group': data_member.gname, 'mtime': data_member.mtime, 'mode': data_member.mode}) + + return _applyQueryOpts(results, queryOpts) + def listRPMFiles(self, rpmID, queryOpts=None): """List files associated with the RPM with the given ID. A list of maps will be returned, each with the following keys: @@ -9310,6 +9763,44 @@ class RootExports(object): return _applyQueryOpts(results, queryOpts) + def getDEBFile(self, debID, filename): + """ + Get info about the file in the given DEB with the given filename. + A map will be returned with the following keys: + - deb_id + - name + - digest + - md5 (alias for digest) + - digest_algo + - size + - flags + + If no such file exists, an empty map will be returned. + """ + deb_info = get_deb(debID) + if not deb_info or not deb_info['build_id']: + return {} + build_info = get_build(deb_info['build_id']) + deb_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.deb(deb_info)) + if not os.path.exists(deb_path): + return {} + + digest_algo = 'md5' # DEB always uses MD5 + flags = 0 # DEB files don't have flags + + debfile = DebFile(deb_path) + + if filename in debfile.md5sums().keys(): + try: + data_member = debfile.data.tgz().getmember(filename) + except KeyError: + data_member = debfile.data.tgz().getmember('./' + filename) + return {'deb_id': deb_info['id'], 'name': filename, 'digest': debfile.md5sums()[filename], + 'digest_algo': digest_algo, 'md5': debfile.md5sums()[filename], 'size': data_member.size, + 'flags': flags, 'user': data_member.uname, 'group': data_member.gname, + 'mtime': data_member.mtime, 'mode': data_member.mode} + return {} + def getRPMFile(self, rpmID, filename): """ Get info about the file in the given RPM with the given filename. @@ -9349,6 +9840,41 @@ class RootExports(object): i += 1 return {} + def getDEBHeaders(self, debID=None, taskID=None, filepath=None, headers=None): + """ + Get the requested headers from the deb. Header names are case-insensitive. + If a header is requested that does not exist an exception will be raised. + Returns a map of header names to values. If the specified ID + is not valid or the deb does not exist on the file system, an empty map + will be returned. + """ + if not headers: + headers = [] + if debID: + deb_info = get_deb(debID) + if not deb_info or not deb_info['build_id']: + return {} + build_info = get_build(deb_info['build_id']) + deb_path = os.path.join(koji.pathinfo.build(build_info), koji.pathinfo.deb(deb_info)) + if not os.path.exists(deb_path): + return {} + elif taskID: + if not filepath: + raise koji.GenericError, 'filepath must be specified with taskID' + if filepath.startswith('/') or '../' in filepath: + raise koji.GenericError, 'invalid filepath: %s' % filepath + deb_path = os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(taskID), + filepath) + else: + raise koji.GenericError, 'either debID or taskID and filepath must be specified' + + debfile = DebFile(deb_path) + result = {} + for header in headers: + result[header] = koji.fixEncoding(debfile.debcontrol()[header]) + return result + def getRPMHeaders(self, rpmID=None, taskID=None, filepath=None, headers=None): """ Get the requested headers from the rpm. Header names are case-insensitive. diff --git a/koji/__init__.py b/koji/__init__.py index b453017..1157f75 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1674,6 +1674,10 @@ class PathInfo(object): """Return the directory where the image for the build are stored""" return self.build(build) + '/images' + def deb(self,debinfo): + """Return the path (relative to build_dir) where a deb belongs""" + return "%(arch)s/%(name)s_%(version)s-%(release)s_%(arch)s.deb" % debinfo + def rpm(self,rpminfo): """Return the path (relative to build_dir) where an rpm belongs""" return "%(arch)s/%(name)s-%(version)s-%(release)s.%(arch)s.rpm" % rpminfo diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 3936055..c493314 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -159,6 +159,43 @@ #end if + + DEBs + + #if $len($debsByArch) > 0 + + #set $arches = $debsByArch.keys() + #silent $arches.sort() + #for $arch in $arches + + + + + #for $deb in $debsByArch[$arch] + + #set $debfile = '%(name)s_%(version)s-%(release)s_%(arch)s.deb' % $deb + #set $debpath = $pathinfo.deb($deb) + + + + #end for + #end for +
$arch + #if $task + #if $arch == 'noarch' + (build logs) + #else + (build logs) + #end if + #end if +
+ $debfile (info) (download) +
+ #else + No DEBs + #end if + + #if $archives Archives diff --git a/www/kojiweb/debinfo.chtml b/www/kojiweb/debinfo.chtml new file mode 100644 index 0000000..5adcaa3 --- /dev/null +++ b/www/kojiweb/debinfo.chtml @@ -0,0 +1,154 @@ +#import koji +#from kojiweb import util +#import time +#import urllib + +#attr _PASSTHROUGH = ['debID', 'fileOrder', 'fileStart', 'buildrootOrder', 'buildrootStart'] + +#include "includes/header.chtml" +

Information for RPM $deb.name-$deb.version-$deb.release.${deb.arch}.deb

+ + + + + + + #if $build + + #else + + #end if + + + #if $build + + #else + + #end if + + + + + + + + #if $deb.external_repo_id == 0 + + + + #end if + #if $build and $build.state == $koji.BUILD_STATES.DELETED + + + + #end if + #if $deb.external_repo_id + + + + #end if + + + + + + + #if $builtInRoot + + + + #end if + #if $deb.external_repo_id == 0 + + + + + + + + + + + + + + + + + #end if +
ID$deb.id
Name$deb.nameName$deb.name
Version$deb.versionVersion$deb.version
Release$deb.release
Arch$deb.arch
Description$util.escapeHTML($description)
Statedeleted
External Repository$deb.external_repo_name
Size$deb.size
Payload Hash$deb.payloadhash
Buildroot$builtInRoot.tag_name-$builtInRoot.id-$builtInRoot.repo_id
Depends + #if $len($Depends) > 0 + + #for $dep in $Depends + + + + #end for +
$dep.name
+ #else + No Depends + #end if +
Recommends + #if $len($Recommends) > 0 + + #for $dep in $Recommends + + + + #end for +
$dep.name
+ #else + No Recommends + #end if +
Suggests + #if $len($Suggests) > 0 + + #for $dep in $Suggests + + + + #end for +
$dep.name
+ #else + No Suggests + #end if +
Files + #if $len($files) > 0 + + + + + + + + + #for $file in $files + + + + #end for +
+ #if $len($filePages) > 1 +
+ Page: + +
+ #end if + #if $fileStart > 0 + <<< + #end if + #echo $fileStart + 1 # through #echo $fileStart + $fileCount # of $totalFiles + #if $fileStart + $fileCount < $totalFiles + >>> + #end if +
Name $util.sortImage($self, 'name', 'fileOrder')Size $util.sortImage($self, 'size', 'fileOrder')
$util.escapeHTML($file.name)$file.size
+ #else + No Files + #end if +
+ +#include "includes/footer.chtml" diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 420c606..9b98393 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1099,6 +1099,8 @@ def buildinfo(environ, buildID): tags.sort(_sortbyname) rpms = server.listBuildRPMs(build['id']) rpms.sort(_sortbyname) + debs = server.listBuildDEBs(build['id']) + debs.sort(_sortbyname) mavenbuild = server.getMavenBuild(buildID) winbuild = server.getWinBuild(buildID) imagebuild = server.getImageBuild(buildID) @@ -1127,6 +1129,7 @@ def buildinfo(environ, buildID): archivesByExt.setdefault(os.path.splitext(archive['filename'])[1][1:], []).append(archive) rpmsByArch = {} + debsByArch = {} debuginfos = [] for rpm in rpms: if koji.is_debuginfo(rpm['name']): @@ -1144,6 +1147,9 @@ def buildinfo(environ, buildID): values['description'] = koji.fixEncoding(headers.get('description')) values['changelog'] = server.getChangelogEntries(build['id']) + for deb in debs: + debsByArch.setdefault(deb['arch'], []).append(deb) + noarch_log_dest = 'noarch' if build['task_id']: task = server.getTaskInfo(build['task_id'], request=True) @@ -1191,6 +1197,7 @@ def buildinfo(environ, buildID): values['build'] = build values['tags'] = tags values['rpmsByArch'] = rpmsByArch + values['debsByArch'] = debsByArch values['task'] = task values['mavenbuild'] = mavenbuild values['winbuild'] = winbuild @@ -1346,6 +1353,38 @@ def userinfo(environ, userID, packageOrder='package_name', packageStart=None, bu return _genHTML(environ, 'userinfo.chtml') +def debinfo(environ, debID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): + values = _initValues(environ, 'DEB Info', 'builds') + server = _getServer(environ) + + debID = int(debID) + deb = server.getDEB(debID) + + values['title'] = '%(name)s-%(version)s-%(release)s.%(arch)s.deb' % deb + ' | DEB Info' + + build = None + if deb['build_id'] != None: + build = server.getBuild(deb['build_id']) + builtInRoot = None + if deb['buildroot_id'] != None: + builtInRoot = server.getBuildroot(deb['buildroot_id']) + if deb['external_repo_id'] == 0: + for dep_type in ['Depends', 'Recommends', 'Suggests', 'Pre-Depends', 'Build-Depends', 'Build-Depends-Indep']: + values[dep_type] = server.getDEBDeps(deb['id'], dep_type) + values[dep_type].sort(_sortbyname) + headers = server.getDEBHeaders(deb['id'], headers=['Description']) + values['description'] = koji.fixEncoding(headers.get('Description')) + + values['debID'] = debID + values['deb'] = deb + values['build'] = build + values['builtInRoot'] = builtInRoot + + files = kojiweb.util.paginateMethod(server, values, 'listDEBFiles', args=[deb['id']], + start=fileStart, dataName='files', prefix='file', order=fileOrder) + + return _genHTML(environ, 'debinfo.chtml') + def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): values = _initValues(environ, 'RPM Info', 'builds') server = _getServer(environ) @@ -1428,12 +1467,13 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO return _genHTML(environ, 'archiveinfo.chtml') -def fileinfo(environ, filename, rpmID=None, archiveID=None): +def fileinfo(environ, filename, rpmID=None, archiveID=None, debID=None): values = _initValues(environ, 'File Info', 'builds') server = _getServer(environ) values['rpm'] = None values['archive'] = None + values['deb'] = None if rpmID: rpmID = int(rpmID) @@ -1453,8 +1493,17 @@ def fileinfo(environ, filename, rpmID=None, archiveID=None): if not file: raise koji.GenericError, 'no file %s in archive %i' % (filename, archiveID) values['archive'] = archive + elif debID: + debID = int(debID) + deb = server.getDEB(debID) + if not deb: + raise koji.GenericError, 'invalid DEB ID: %i' % debID + file = server.getDEBFile(deb['id'], filename) + if not file: + raise koji.GenericError, 'no file %s in DEB %i' % (filename, debID) + values['deb'] = deb else: - raise koji.GenericError, 'either rpmID or archiveID must be specified' + raise koji.GenericError, 'either rpmID, archiveID or debID must be specified' values['title'] = file['name'] + ' | File Info' @@ -2197,6 +2246,7 @@ _DEFAULT_SEARCH_ORDER = { # For searches against large tables, use '-id' to show most recent first 'build' : '-id', 'rpm' : '-id', + 'deb' : '-id', 'maven' : '-id', 'win' : '-id', # for other tables, ordering by name makes much more sense