From b09fd01677eaf38422db2805ba16254c7a3d0785 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Dec 19 2019 12:55:06 +0000 Subject: additional options to clean database Fixes: https://pagure.io/koji/issue/1707 --- diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 265b5c0..6c4283f 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -3,46 +3,163 @@ import os from optparse import OptionParser +from six.moves import xmlrpc_client from six.moves.configparser import RawConfigParser import koji.db -def clean_sessions(cursor, vacuum): - q = " FROM sessions WHERE update_time < now() - '1 day'::interval" +def clean_sessions(cursor, vacuum, test, age): + q = " FROM sessions WHERE update_time < NOW() - '%s days'::interval" % int(age) if options.verbose: cursor.execute("SELECT COUNT(*) " + q) rows = cursor.fetchall()[0][0] print("Deleting %d sessions" % rows) - cursor.execute("DELETE " + q) - if vacuum: - cursor.execute("VACUUM ANALYZE sessions") + if not test: + cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE sessions") -def clean_reservations(cursor, vacuum): - q = " FROM build_reservations WHERE created < now() - '1 day'::interval" +def clean_reservations(cursor, vacuum, test, age): + q = " FROM build_reservations WHERE created < NOW() - '%s days'::interval" % int(age) if options.verbose: cursor.execute("SELECT COUNT(*) " + q) rows = cursor.fetchall()[0][0] print("Deleting %d build reservations" % rows) - cursor.execute("DELETE " + q) + if not test: + cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE build_reservations") + + +def clean_notification_tasks(cursor, vacuum, test, age): + q = " FROM task WHERE method = 'build' AND completion_time < NOW() - '%s days'::interval" % int(age) + if options.verbose: + cursor.execute("SELECT COUNT(*) " + q) + rows = cursor.fetchall()[0][0] + print("Deleting %d tagNotification tasks" % rows) + + if not test: + # cascade + cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE task") + + +def clean_scratch_tasks(cursor, vacuum, test, age): + q = """ FROM task + WHERE method = 'build' AND + completion_time < NOW() - '%s days'::interval AND + request LIKE '%%%%scratch%%%%'""" % int(age) + if options.verbose: + cursor.execute("SELECT COUNT(*) " + q) + rows = cursor.fetchall()[0][0] + print("Deleting %d scratch build tasks" % rows) + + if test: + return + + # we should check, that it is really a scratch build + ids = [] + # will be dropped automatically in the end of script/connection + cursor.execute("CREATE TEMPORARY TABLE temp_scratch_tasks (task_id INTEGER NOT NULL)") + cursor.execute("SELECT id, request " + q) + for row in cursor.fetchall(): + task_id, request = row + try: + params, method = xmlrpc_client.loads(request) + opts = params[2] + if opts['scratch']: + cursor.execute("INSERT INTO temp_scratch_tasks VALUES (%s)", (task_id,)) + ids.append(task_id) + except: + continue + + parents = ids + while True: + if not parents: + break + children = [] + cursor.execute("SELECT id FROM task WHERE parent IN %s", (parents,)) + for row in cursor.fetchall(): + children.append(row[0]) + parents = children + if children: + values = ', '.join(["(%d)" % task_id for task_id in children]) + cursor.execute("INSERT INTO temp_scratch_tasks VALUES %s" % values) + + if not ids: + return + + # delete standard buildroots + cursor.execute("DELETE FROM standard_buildroot WHERE task_id IN (SELECT task_id FROM temp_scratch_tasks)") + + # delete tasks finally + cursor.execute("DELETE FROM task WHERE id IN (SELECT task_id FROM temp_scratch_tasks)") + if vacuum: - cursor.execute("VACUUM ANALYZE build_reservations") + cursor.execute("VACUUM ANALYZE task") + + +def clean_buildroots(cursor, vacuum, test): + q = " FROM buildroot WHERE cg_id IS NULL AND id NOT IN (SELECT buildroot_id FROM standard_buildroot)" + + if options.verbose: + cursor.execute("SELECT COUNT(*) " + q) + rows = cursor.fetchall()[0][0] + print("Deleting %d buildroots" % rows) + + if not test: + cursor.execute("DELETE FROM buildroot_listing WHERE buildroot_id IN (SELECT id %s)" % q) + cursor.execute("DELETE " + q) + if vacuum: + cursor.execute("VACUUM ANALYZE build_reservations") if __name__ == "__main__": global options parser = OptionParser("%prog cleans koji database") parser.add_option('-v', '--verbose', action="store_true", help="Be verbose") + parser.add_option('-t', '--test', action="store_true", + help="Don't delete anything, print estimations (implies -v)") parser.add_option('-c', '--conf', default='/etc/koji-hub/hub.conf', action='store', help="Path to koji's hub.conf") parser.add_option('--no-vacuum', action="store_false", dest="vacuum", default=True, help="Don't run vacuum on affected tables") + parser.add_option('--sessions-age', type=int, + action="store", default=1, metavar="DAYS", + help="Delete sessions older than this (default: 1 day)") + parser.add_option('--reservations-age', type=int, + action="store", default=1, metavar="DAYS", + help="Delete CG reservations older than this (default: 1 day)") + parser.add_option('--tag-notifications', action="store_true", + help="Delete tagNotification tasks", default=False) + parser.add_option('--tag-notifications-age', type=int, + action="store", default=730, metavar="DAYS", + help="Delete tagNotification tasks older than this (default: 2 years)") + parser.add_option("--scratch-builds", action="store_true", + dest="scratch", default=False, + help="Delete scratch build tasks") + parser.add_option('--scratch-builds-age', type=int, dest="scratch_age", + action="store", default=730, metavar="DAYS", + help="Delete scratch builds' tasks older than this (default: 2 years") + parser.add_option('--buildroots', action="store_true", + help="Delete unreferenced buildroots") + parser.add_option('-f', '--force', action="store_true", + help="This is needs for all options affecting auditable data") options, args = parser.parse_args() + if options.test: + options.verbose = True + print("Running in test mode, no changes will be made") + + if not options.force and (options.tag_notifications or options.scratch or options.buildroots): + parser.error("You need to pass --force to trigger these actions") + if not os.path.exists(options.conf): parser.error("Config file doesn't exist") @@ -85,6 +202,11 @@ if __name__ == "__main__": conn.set_session(autocommit=True) cursor = conn.cursor() - clean_sessions(cursor, options.vacuum) - clean_reservations(cursor, options.vacuum) - conn.commit() + clean_sessions(cursor, options.vacuum, options.test, options.sessions_age) + clean_reservations(cursor, options.vacuum, options.test, options.reservations_age) + if options.tag_notifications: + clean_notification_tasks(cursor, options.vacuum, options.test, age=options.tag_notifications_age) + if options.scratch: + clean_scratch_tasks(cursor, options.vacuum, options.test, age=options.scratch_age) + if options.buildroots: + clean_buildroots(cursor, options.vacuum, options.test)