From 5adbae33557a48164c110a2ca4037b6529d98f0b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 26 2016 09:25:42 +0000 Subject: [PATCH 1/5] Add a script to remove the cache of one or more widgets as needed --- diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py new file mode 100755 index 0000000..b1b32c5 --- /dev/null +++ b/smart_cache_invalidator.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +""" +Small script to remove the cache of one or more widgets while keeping the +other ones un-touched. + +This is really useful for development purposes as it allow seeing changes +made to a widget without dropping the entire cache database. +""" + + +import argparse +import os + +import anydbm +import fedmsg.config +import fedmsg.meta + +import hubs.models + +# get the DB session +fedmsg_config = fedmsg.config.load_config() +fedmsg.meta.make_processors(**fedmsg_config) + +session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri']) + + +def do_list(args): + ''' List the different widget for which there is data cached. ''' + db = anydbm.open(args.cache_db) + for key in db.keys(): + widget_id = key.split('|')[0] + widget = hubs.models.Widget.get(session, widget_id) + if widget: + print '- Widget cached %s (#%s) in %s' % ( + widget.hub_id, widget_id, widget.plugin) + db.close() + + +def do_clean(args): + ''' Clean the widget for which there is data cached. ''' + for ext in ['.dogpile.lock', '.rw.lock']: + filename = args.cache_db + ext + if os.path.exists(filename): + os.unlink(filename) + + db = anydbm.open(args.cache_db, 'w') + for key in db.keys(): + widget_id = key.split('|')[0] + widget = hubs.models.Widget.get(session, widget_id) + if args.widget in [widget.plugin if widget else '', widget_id]: + del(db[key]) + break + db.close() + + +def setup_parser(): + ''' + Set the main arguments. + ''' + parser = argparse.ArgumentParser(prog="smart_cache_invalidator") + # General connection options + parser.add_argument('--cache-db', dest="cache_db", + default='/var/tmp/fedora-hubs-cache.db', + help="The Fedora-hubs cache database") + + subparsers = parser.add_subparsers(title='actions') + + # List + parser_list = subparsers.add_parser( + 'list', + help='List the different widgets for which there is cached data') + parser_list.set_defaults(func=do_list) + + # Clean + parser_clean = subparsers.add_parser( + 'clean', help='Clean the specified widget') + parser_clean.add_argument( + 'widget', help="Identifier or name of the widget to clean") + parser_clean.set_defaults(func=do_clean) + + return parser + + +def main(): + ''' Main function ''' + return_code = 0 + + # Set up parser for global args + parser = setup_parser() + + # Parse the commandline + try: + arg = parser.parse_args() + except argparse.ArgumentTypeError as err: + print("\nError: {0}".format(err)) + return 1 + + if not os.path.exists(arg.cache_db): + print('No cache DB found') + return 2 + + try: + arg.func(arg) + except KeyboardInterrupt: + print("\nInterrupted by user.") + return_code = 3 + except argparse.ArgumentError as err: + print('{0}'.format(err.message)) + return_code = 4 + except Exception as err: + print('Error: {0}'.format(err)) + return_code = 5 + + return return_code + + +if __name__ == '__main__': + main() From 175998a6ef9a426ed561ddf1747f074e3b74c2d6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 26 2016 09:25:42 +0000 Subject: [PATCH 2/5] Fix printing using the print function --- diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py index b1b32c5..ee8f298 100755 --- a/smart_cache_invalidator.py +++ b/smart_cache_invalidator.py @@ -8,6 +8,7 @@ This is really useful for development purposes as it allow seeing changes made to a widget without dropping the entire cache database. """ +from __future__ import print_function import argparse import os @@ -32,8 +33,8 @@ def do_list(args): widget_id = key.split('|')[0] widget = hubs.models.Widget.get(session, widget_id) if widget: - print '- Widget cached %s (#%s) in %s' % ( - widget.hub_id, widget_id, widget.plugin) + print('- Widget cached {0} (#{1}) in {2}'.format( + widget.hub_id, widget_id, widget.plugin)) db.close() From d102cc4c4c8f167bab1fed3356f3f3764009de27 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 26 2016 09:25:42 +0000 Subject: [PATCH 3/5] Allow cleaning the cache of multiple widgets at once --- diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py index ee8f298..7e53851 100755 --- a/smart_cache_invalidator.py +++ b/smart_cache_invalidator.py @@ -46,12 +46,15 @@ def do_clean(args): os.unlink(filename) db = anydbm.open(args.cache_db, 'w') - for key in db.keys(): - widget_id = key.split('|')[0] - widget = hubs.models.Widget.get(session, widget_id) - if args.widget in [widget.plugin if widget else '', widget_id]: - del(db[key]) - break + for widget in args.widgets: + for key in db.keys(): + widget_id = key.split('|')[0] + wid_obj = hubs.models.Widget.get(session, widget_id) + if widget in [wid_obj.plugin if wid_obj else '', widget_id]: + print('- Removing cached {0} (#{1}) in {2}'.format( + widget.hub_id, widget_id, widget.plugin)) + del(db[key]) + break db.close() @@ -77,7 +80,8 @@ def setup_parser(): parser_clean = subparsers.add_parser( 'clean', help='Clean the specified widget') parser_clean.add_argument( - 'widget', help="Identifier or name of the widget to clean") + 'widgets', nargs="+", + help="Identifier or name of the one or moe widgets to clean") parser_clean.set_defaults(func=do_clean) return parser From 79adf552ad245cdee178816e7fed5e6df7fd5bf7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 26 2016 09:25:42 +0000 Subject: [PATCH 4/5] Adjust the smart_cache_invalidator script to rely on the base widget The methods present in the base widget allow easy interacting with the cache database without having to do anything manually. --- diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py index 7e53851..08737ee 100755 --- a/smart_cache_invalidator.py +++ b/smart_cache_invalidator.py @@ -13,11 +13,13 @@ from __future__ import print_function import argparse import os -import anydbm +import dogpile.cache import fedmsg.config import fedmsg.meta + import hubs.models +import hubs.widgets.base # get the DB session fedmsg_config = fedmsg.config.load_config() @@ -28,34 +30,26 @@ session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri']) def do_list(args): ''' List the different widget for which there is data cached. ''' - db = anydbm.open(args.cache_db) - for key in db.keys(): - widget_id = key.split('|')[0] - widget = hubs.models.Widget.get(session, widget_id) - if widget: + for widget in session.query(hubs.models.Widget).all(): + key = hubs.widgets.base.cache_key_generator(widget, **widget.config) + result = hubs.widgets.base.cache.get(key, ignore_expiration=True) + if not isinstance(result, dogpile.cache.api.NoValue): print('- Widget cached {0} (#{1}) in {2}'.format( - widget.hub_id, widget_id, widget.plugin)) - db.close() + widget.hub_id, widget.idx, widget.plugin)) def do_clean(args): ''' Clean the widget for which there is data cached. ''' - for ext in ['.dogpile.lock', '.rw.lock']: - filename = args.cache_db + ext - if os.path.exists(filename): - os.unlink(filename) - - db = anydbm.open(args.cache_db, 'w') for widget in args.widgets: - for key in db.keys(): - widget_id = key.split('|')[0] - wid_obj = hubs.models.Widget.get(session, widget_id) - if widget in [wid_obj.plugin if wid_obj else '', widget_id]: - print('- Removing cached {0} (#{1}) in {2}'.format( - widget.hub_id, widget_id, widget.plugin)) - del(db[key]) - break - db.close() + wid_obj = hubs.models.Widget.get(session, widget) + if not wid_obj: + wid_obj = session.query(cls).filter_by(plugin=widget).first() + if not wid_obj: + print('No widget found for {0}'.format(widget)) + + print('- Removing cached {0} (#{1}) in {2}'.format( + wid_obj.hub_id, wid_obj.idx, wid_obj.plugin)) + result = hubs.widgets.base.invalidate_cache(wid_obj, **wid_obj.config) def setup_parser(): From 7240c9b397759610a8d3b749126004fdfaa7be14 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 26 2016 09:25:42 +0000 Subject: [PATCH 5/5] Simplify the arguments accepted by the script We can do this now that we rely on hubs itself to point us to where the cache is. --- diff --git a/smart_cache_invalidator.py b/smart_cache_invalidator.py index 08737ee..77021fc 100755 --- a/smart_cache_invalidator.py +++ b/smart_cache_invalidator.py @@ -42,8 +42,11 @@ def do_clean(args): ''' Clean the widget for which there is data cached. ''' for widget in args.widgets: wid_obj = hubs.models.Widget.get(session, widget) + if not wid_obj: - wid_obj = session.query(cls).filter_by(plugin=widget).first() + wid_obj = session.query(hubs.models.Widget).filter_by( + plugin=widget).first() + if not wid_obj: print('No widget found for {0}'.format(widget)) @@ -57,11 +60,6 @@ def setup_parser(): Set the main arguments. ''' parser = argparse.ArgumentParser(prog="smart_cache_invalidator") - # General connection options - parser.add_argument('--cache-db', dest="cache_db", - default='/var/tmp/fedora-hubs-cache.db', - help="The Fedora-hubs cache database") - subparsers = parser.add_subparsers(title='actions') # List @@ -95,21 +93,14 @@ def main(): print("\nError: {0}".format(err)) return 1 - if not os.path.exists(arg.cache_db): - print('No cache DB found') - return 2 - try: arg.func(arg) except KeyboardInterrupt: print("\nInterrupted by user.") - return_code = 3 - except argparse.ArgumentError as err: - print('{0}'.format(err.message)) - return_code = 4 + return_code = 2 except Exception as err: print('Error: {0}'.format(err)) - return_code = 5 + return_code = 3 return return_code