From 06785343351660d7550d42cf522ec31a0a89b4bd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 12 2019 09:07:47 +0000 Subject: [PATCH 1/5] Small improvements to the dev-data.py script This script is used by new contributors to pagure to create or populate the database. This commit moves the database initiation to the code used within pagure to reduce code duplication. This commit also allows specifying an user name and user email using environment variables to by-pass the logic asking for the user's input which will allow testing this script in pagure's own test suite. Finally this commit also moves deleting all the data in the database at the end of the ``--all`` so it does: create, populate, delete rather than create, delete, populate. Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev-data.py b/dev-data.py index 6f0ef27..087bb3e 100644 --- a/dev-data.py +++ b/dev-data.py @@ -34,34 +34,16 @@ python dev-data.py --all _config = pagure.config.reload_config() -def init_database(): - DB_URL = _config['DB_URL'] - - # create the table if it doesnt exist - pagure.lib.model.create_tables( - DB_URL, - _config.get('PATH_ALEMBIC_INI', None), - acls=_config.get('ACLS', {}), - debug=True) - - engine = create_engine('%s' % DB_URL, echo=True) - - metadata = MetaData(engine) - metadata.reflect(bind=engine) - return engine, metadata - - -def empty_dev_db(metadata, engine): +def empty_dev_db(session): print('') - print('') - print('WARNING: Deleting all data from ', _config['DB_URL']) - # Dangerous: this will wipe the data from the table but keep the schema - print('') - response = six.moves.input('Do you want to continue? (yes/no) ') - if 'yes'.startswith(response.lower()): - for tbl in reversed(metadata.sorted_tables): - if tbl.fullname != 'acls': - engine.execute(tbl.delete()) + print('WARNING: Deleting all data from', _config['DB_URL']) + response = os.environ.get("FORCE_DELETE") + if not response: + response = six.moves.input('Do you want to continue? (yes/no) ') + if response.lower().startswith('y'): + tables = reversed(pagure.lib.model_base.BASE.metadata.sorted_tables) + for tbl in tables: + session.execute(tbl.delete()) else: exit("Aborting.") @@ -70,10 +52,6 @@ def insert_data(session, username, user_email): _config['EMAIL_SEND'] = False _config['TESTING'] = True - # Populate with default statuses - create_default_status(session) - print('Default statuses populated') - ###################################### # tags item = pagure.lib.model.Tag( @@ -430,7 +408,7 @@ def insert_data(session, username, user_email): session.commit() -##################################### + ##################################### # tokens tests.create_tokens(session, user_id=pingou.id, project_id=project1.id) @@ -668,6 +646,46 @@ def add_content_git_repo(folder, branch='master'): shutil.rmtree(newfolder) +def _get_username(): + invalid_option = ['pingou', 'foo'] + user_name = os.environ.get("USER_NAME") + if not user_name: + print("") + user_name = six.moves.input( + "Enter your username so we can add you into the test data: ") + cnt = 0 + while not user_name.strip() or user_name in invalid_option: + print("Reserved names: " + str(invalid_option)) + user_name = six.moves.input( + "Enter your username so we can add you into the " + "test data: ") + cnt += 1 + if cnt == 4: + print("We asked too many times, bailing") + sys.exit(1) + + return user_name + + +def _get_user_email(): + invalid_option = ['bar@pingou.com', 'foo@bar.com'] + user_email = os.environ.get("USER_EMAIL") + if not user_email: + print("") + user_email = six.moves.input("Enter your user email: ") + + cnt = 0 + while not user_email.strip() or user_email in invalid_option: + print("Reserved names: " + str(invalid_option)) + user_email = six.moves.input("Enter your user email: ") + cnt += 1 + if cnt == 4: + print("We asked too many times, bailing") + sys.exit(1) + + return user_email + + if __name__ == "__main__": desc = "Run the dev database initialization/insertion/deletion " \ "script for db located " + str(_config['DB_URL']) @@ -679,7 +697,7 @@ if __name__ == "__main__": parser.add_argument('-d', '--delete', action="store_true", help="Wipe the dev db") parser.add_argument('-a', '--all', action="store_true", - help="Create, Wipe, Populate the dev db") + help="Create, Populate then Wipe the dev db") args = parser.parse_args() @@ -687,34 +705,23 @@ if __name__ == "__main__": if not any(vars(args).values()): parser.error('No arguments provided.') - if args.init or args.delete or args.all: - eng, meta = init_database() + session = None - if args.delete or args.all: - empty_dev_db(meta, eng) + if args.init or args.all: + session = pagure.lib.model.create_tables( + db_url=_config["DB_URL"], + alembic_ini=None, + acls=_config["ACLS"], + debug=False) + print("Database created") if args.populate or args.all: - session = pagure.lib.model_base.create_session(_config['DB_URL']) - invalid_option = ['pingou', 'bar@pingou.com', 'foo', 'foo@bar.com'] - print("") - user_name = six.moves.input( - "Enter your username so we can add you into the test data: ") - while user_name in invalid_option: - print("Reserved names: " + str(invalid_option)) - user_name = six.moves.input( - "Enter your username so we can add you into the test data: ") - - if not user_name.replace(" ", ""): - user_name = 'pythagoras' - - print("") - user_email = six.moves.input("Enter your user email: ") - - while user_email in invalid_option: - print("Reserved names: " + str(invalid_option)) - user_email = six.moves.input("Enter your user email: ") - - if not user_email.replace(" ", ""): - user_email = 'pythagoras@math.com' + if not session: + session = pagure.lib.query.create_session(_config['DB_URL']) + user_name = _get_username() + user_email = _get_user_email() insert_data(session, user_name, user_email) + + if args.delete or args.all: + empty_dev_db(session) From 6e31e0a82fb3a07ffbc7819c75b99c0c93ec1cea Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 12 2019 09:07:47 +0000 Subject: [PATCH 2/5] Add unit-tests checking the behavior of the dev-data.py utility script Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_dev_data.py b/tests/test_dev_data.py new file mode 100644 index 0000000..b60c734 --- /dev/null +++ b/tests/test_dev_data.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +""" + (c) 2019 - Copyright Red Hat Inc + + Authors: + Pierre-Yves Chibon + +""" + +from __future__ import unicode_literals, absolute_import + +import os +import subprocess +import sys +import unittest + +import six + + +REPO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + +import tests # noqa + + +class TestDevData(tests.SimplePagureTest): + """This test class contains tests pertaining to the dev-data utility + script.""" + + maxDiff = None + + def test_dev_data_all(self): + """Check how dev-data --all performs + + """ + + config_path = os.path.join(self.path, "config") + with open(config_path, "w") as f: + f.write("DB_URL = 'sqlite:///%s/db_dev_data.sqlite'" % self.path) + + env = { + "USER_NAME": "testuser", + "USER_EMAIL": "testuser@example.com", + "FORCE_DELETE": "yes", + "PAGURE_CONFIG": config_path, + } + proc1 = subprocess.Popen( + [sys.executable, "dev-data.py", "--all"], + cwd=REPO_PATH, + stdout=subprocess.PIPE, + env=env, + ) + stdout, stderr = proc1.communicate() + if isinstance(stdout, six.binary_type): + stdout = stdout.decode("utf-8") + output = ( + """Database created +User created: pingou , testing123 +User created: foo , testing123 +User created: testuser , testing123 +Created "admin" group. Pingou is a member. +Created "group" group. Pingou is a member. +Created "rel-eng" group. Pingou is a member. +git folder already deleted +docs folder already deleted +tickets folder already deleted +requests folder already deleted + +WARNING: Deleting all data from sqlite:///%s/db_dev_data.sqlite +""" + % self.path + ) + + self.assertEqual(len(stdout.split("\n")), 14) + self.assertEqual(stdout, output) + + def test_dev_data_delete(self): + """Check how dev-data --init --delete performs + + """ + + config_path = os.path.join(self.path, "config") + + env = { + "USER_NAME": "testuser", + "USER_EMAIL": "testuser@example.com", + "FORCE_DELETE": "yes", + "PAGURE_CONFIG": config_path, + } + proc1 = subprocess.Popen( + [sys.executable, "dev-data.py", "--init", "--delete"], + cwd=REPO_PATH, + stdout=subprocess.PIPE, + env=env, + ) + stdout, stderr = proc1.communicate() + if isinstance(stdout, six.binary_type): + stdout = stdout.decode("utf-8") + output = ( + """Database created + +WARNING: Deleting all data from %s +""" + % self.dbpath + ) + + self.assertEqual(len(stdout.split("\n")), 4) + self.assertEqual(stdout.split("\n"), output.split("\n")) + + def test_dev_data_init(self): + """Check how dev-data --init performs + + """ + + config_path = os.path.join(self.path, "config") + + env = { + "USER_NAME": "testuser", + "USER_EMAIL": "testuser@example.com", + "FORCE_DELETE": "yes", + "PAGURE_CONFIG": config_path, + } + proc1 = subprocess.Popen( + [sys.executable, "dev-data.py", "--init"], + cwd=REPO_PATH, + stdout=subprocess.PIPE, + env=env, + ) + stdout, stderr = proc1.communicate() + if isinstance(stdout, six.binary_type): + stdout = stdout.decode("utf-8") + output = "Database created\n" + + self.assertEqual(len(stdout.split("\n")), 2) + self.assertEqual(stdout.split("\n"), output.split("\n")) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 8782b5924f7b8d6f994c6936ad6e7b52a3e7927a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 12 2019 09:07:47 +0000 Subject: [PATCH 3/5] Small code formatting change Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/__init__.py b/tests/__init__.py index c21ac3d..db932fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -373,7 +373,8 @@ class SimplePagureTest(unittest.TestCase): # Write a config file config_values = { - 'path': self.path, 'dburl': self.dbpath, + 'path': self.path, + 'dburl': self.dbpath, 'enable_docs': True, 'docs_folder': '%s/repos/docs' % self.path, 'enable_tickets': True, From 30c623f443018675ef71ece82dbaddc40f100a09 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 12 2019 11:27:32 +0000 Subject: [PATCH 4/5] Fix keeping in memory the config values to be accessible for all tests Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/__init__.py b/tests/__init__.py index db932fa..6a61cf2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -390,10 +390,11 @@ class SimplePagureTest(unittest.TestCase): 'repospanner_admin_migration': 'False', } config_values.update(self.config_values) + self.config_values = config_values config_path = os.path.join(self.path, 'config') if not os.path.exists(config_path): with open(config_path, 'w') as f: - f.write(CONFIG_TEMPLATE % config_values) + f.write(CONFIG_TEMPLATE % self.config_values) os.environ["PAGURE_CONFIG"] = config_path pagure_config.update(reload_config()) From c30238cfd76e5094f58d73dece68eaa12259e5af Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Feb 12 2019 11:27:32 +0000 Subject: [PATCH 5/5] Fix getting the dev-data tests to run This sets up redis and makes it accessible to the dev-data script, in addition, the git folders are pointed to the temp folder used for the tests so this doesn't impact our own dev environment. Signed-off-by: Pierre-Yves Chibon --- diff --git a/tests/test_dev_data.py b/tests/test_dev_data.py index b60c734..720616c 100644 --- a/tests/test_dev_data.py +++ b/tests/test_dev_data.py @@ -23,7 +23,7 @@ REPO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) import tests # noqa -class TestDevData(tests.SimplePagureTest): +class TestDevData(tests.Modeltests): """This test class contains tests pertaining to the dev-data utility script.""" @@ -36,7 +36,12 @@ class TestDevData(tests.SimplePagureTest): config_path = os.path.join(self.path, "config") with open(config_path, "w") as f: - f.write("DB_URL = 'sqlite:///%s/db_dev_data.sqlite'" % self.path) + f.write("DB_URL = 'sqlite:///%s/db_dev_data.sqlite'\n" % self.path) + f.write("GIT_FOLDER = '%s/repos'\n" % self.path) + f.write( + "BROKER_URL = 'redis+socket://%(global_path)s/broker'\n" % \ + self.config_values) + f.write("CELERY_CONFIG = {'task_always_eager': True}\n") env = { "USER_NAME": "testuser",