From d79d45d9515093a23eb629ef364d58fa8d0a0dbf Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 25 2020 14:17:31 +0000 Subject: [PATCH 1/3] Limit final query by prechecking buildroot ids Fixes: https://pagure.io/koji/issue/1562 --- diff --git a/hub/kojihub.py b/hub/kojihub.py index fa13a4a..33e029d 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5323,15 +5323,48 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= clauses.append('standard_buildroot.state IN %(state)s') else: clauses.append('standard_buildroot.state = %(state)i') + + # following filters can dramatically limit overall query size + # run separate queries for picking smallest candidate set + candidate_buildroot_ids = set() if rpmID is not None: joins.insert(0, 'buildroot_listing ON buildroot.id = buildroot_listing.buildroot_id') fields.append(('buildroot_listing.is_update', 'is_update')) clauses.append('buildroot_listing.rpm_id = %(rpmID)i') + query = QueryProcessor(columns=['buildroot_id'], tables=['buildroot_listing'], + clauses=['rpm_id = %(rpmID)i'], opts={'asList': True}, + values=locals()) + result = query.execute() + if candidate_buildroot_ids: + candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) + else: + candidate_buildroot_ids = set(result) + if archiveID is not None: joins.append('buildroot_archives ON buildroot.id = buildroot_archives.buildroot_id') clauses.append('buildroot_archives.archive_id = %(archiveID)i') + query = QueryProcessor(columns=['buildroot_id'], tables=['buildroot_archives'], + clauses=['archive_id = %(archiveID)i'], opts={'asList': True}, + values=locals()) + result = query.execute() + if candidate_buildroot_ids: + candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) + else: + candidate_buildroot_ids |= set(result) if taskID is not None: clauses.append('standard_buildroot.task_id = %(taskID)i') + query = QueryProcessor(columns=['buildroot_id'], tables=['standard_buildroot'], + clauses=['task_id = %(taskID)i'], opts={'asList': True}, + values=locals()) + result = query.execute() + if candidate_buildroot_ids: + candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) + else: + candidate_buildroot_ids |= set(result) + + if candidate_buildroot_ids: + candidate_buildroot_ids = list(candidate_buildroot_ids) + clauses.append('buildroot.id IN %(candidate_buildroot_ids)s') query = QueryProcessor(columns=[f[0] for f in fields], aliases=[f[1] for f in fields], tables=tables, joins=joins, clauses=clauses, values=locals(), From ce1753159bd420fb9da8a99148f85e6ff3a8551a Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 25 2020 14:17:31 +0000 Subject: [PATCH 2/3] move inner join before outer joins --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 33e029d..1af070a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5341,7 +5341,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= candidate_buildroot_ids = set(result) if archiveID is not None: - joins.append('buildroot_archives ON buildroot.id = buildroot_archives.buildroot_id') + joins.insert(0, 'buildroot_archives ON buildroot.id = buildroot_archives.buildroot_id') clauses.append('buildroot_archives.archive_id = %(archiveID)i') query = QueryProcessor(columns=['buildroot_id'], tables=['buildroot_archives'], clauses=['archive_id = %(archiveID)i'], opts={'asList': True}, From 9906c7d3dc08dbc8f834416e7bb3b292bf578af1 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mar 25 2020 14:24:23 +0000 Subject: [PATCH 3/3] use same set operators --- diff --git a/hub/kojihub.py b/hub/kojihub.py index 1af070a..06fadb8 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -5334,11 +5334,8 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= query = QueryProcessor(columns=['buildroot_id'], tables=['buildroot_listing'], clauses=['rpm_id = %(rpmID)i'], opts={'asList': True}, values=locals()) - result = query.execute() - if candidate_buildroot_ids: - candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) - else: - candidate_buildroot_ids = set(result) + result = set(query.execute()) + candidate_buildroot_ids = result if archiveID is not None: joins.insert(0, 'buildroot_archives ON buildroot.id = buildroot_archives.buildroot_id') @@ -5346,21 +5343,22 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= query = QueryProcessor(columns=['buildroot_id'], tables=['buildroot_archives'], clauses=['archive_id = %(archiveID)i'], opts={'asList': True}, values=locals()) - result = query.execute() + result = set(query.execute()) if candidate_buildroot_ids: - candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) + candidate_buildroot_ids &= result else: - candidate_buildroot_ids |= set(result) + candidate_buildroot_ids = result + if taskID is not None: clauses.append('standard_buildroot.task_id = %(taskID)i') query = QueryProcessor(columns=['buildroot_id'], tables=['standard_buildroot'], clauses=['task_id = %(taskID)i'], opts={'asList': True}, values=locals()) - result = query.execute() + result = set(query.execute()) if candidate_buildroot_ids: - candidate_buildroot_ids = candidate_buildroot_ids.intersection(set(result)) + candidate_buildroot_ids &= result else: - candidate_buildroot_ids |= set(result) + candidate_buildroot_ids = result if candidate_buildroot_ids: candidate_buildroot_ids = list(candidate_buildroot_ids)