From 269c9386b17030d2670091a8845e1e9bebfc80ba Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Aug 06 2019 12:30:44 +0000 Subject: Add dev_scripts/find_images_to_rebuild.py and docs. --- diff --git a/dev_scripts/README.md b/dev_scripts/README.md index 3b7065b..14e2e98 100644 --- a/dev_scripts/README.md +++ b/dev_scripts/README.md @@ -6,8 +6,5 @@ It is hard to test Freshmaker end to end locally, because it depends on lot of other services. To be able to test some parts locally, we use few scripts stored in this directory. -The scripts are just templates creating instances of particular Freshmaker -classes so they can later be used for testing. - -The intended use-case is that when developing some feature, you can test -the feature using these scripts. You should *not* commit the changed script. +The scripts with "template_" prefix are just templates creating instances +of particular Freshmaker classes so they can later be used for testing. diff --git a/dev_scripts/config.py.template b/dev_scripts/config.py.template new file mode 100644 index 0000000..c55900c --- /dev/null +++ b/dev_scripts/config.py.template @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- + +import os +import tempfile + +from freshmaker.config import all_, any_ # noqa + +dbdir = os.path.abspath(os.path.dirname(__file__)) + +class BaseConfiguration(object): + + # EDIT FOLLOWING OPTIONS BASED ON YOUR ENVIRONMENT: + # ================================================= + # + # Base URL of git repository with source artifacts. + GIT_BASE_URL = "git://pkgs.fedoraproject.org" + + # Read Koji configuration from profile instead of reading them from + # configuration file directly. + KOJI_PROFILE = "brew" + + ERRATA_TOOL_SERVER_URL = "https://errata.localhost.tld" + PULP_SERVER_URL = "https://pulp.localhost.tld" + PULP_USERNAME = "username" + PULP_PASSWORD = "password" + LIGHTBLUE_SERVER_URL = "https://lightblue.localhost.tld" + LIGHTBLUE_CERTIFICATE = "./lb-freshmaker-cert.pem" + LIGHTBLUE_PRIVATE_KEY = "./lb-freshmaker-key.pem" + LIGHTBLUE_RELEASED_DEPENDENCIES_ONLY = True + + HANDLER_BUILD_WHITELIST = { + 'global': { + 'image': any_( + # In DRY_RUN mode, allow rebuilding any shipped-live advisory. + # That's useful for testing when submitting rebuilds manually. + { + 'advisory_state': 'SHIPPED_LIVE', + 'dry_run': True, + 'published': True, + } + ) + }, + } + + + # DEFAULT VALUES: + # =============== + # Make this random (used to generate session keys) + SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0' + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(os.path.join( + dbdir, 'freshmaker.db')) + SQLALCHEMY_TRACK_MODIFICATIONS = False + + HOST = '0.0.0.0' + PORT = 5001 + + SERVER_NAME = 'localhost:5001' + + DEBUG = True + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = 'koji' + + # Available backends are: console, file, journal. + LOG_BACKEND = 'console' + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = 'debug' + + # Read Koji configuration from profile instead of reading them from + # configuration file directly. + KOJI_PROFILE = "brew" + + # Settings for docker image rebuild handler + KOJI_CONTAINER_SCRATCH_BUILD = False + + SSL_ENABLED = False + + # whitelist for handlers to decide whether an artifact + # can be built. + # + # In format of: + # + # { : + # { : } + # } + # + # The `handler_name` is usually set to "global" to affect all + # the handlers. + # + # The `rule(s)` part of a whitelist are dictionaries with key named as + # some artifact attribute. The value can be str, bool or list of strings. + # If it is list of strings, the rule matches if any string from the list + # matches the artifact attribute. + # + # The rule(s) can be also grouped using the any_() or all_() functions: + # + # - The any_(rule_1, rule_2, ...) matches when any of the rules + # matches. + # - The all_(rule_1, rule_2, ...) matches when all the rules matches. + # + # For more information see . + # + # Here is an example of allowing container images to be build as soon as + # an RHSA advisory with critical/important severity or with hightouch bug + # moves to SHIPPED_LIVE: + # + # HANDLER_BUILD_WHITELIST = { + # "global": { + # "image": all_( + # {'advisory_name': 'RHSA-.*' + # 'advisory_state: 'SHIPPED_LIVE'}, + # any_( + # {'has_hightouch_bugs': True}, + # {'severity': ['critical', 'important']} + # ) + # ) + # }, + # } + + # whitelist for handlers to decide whether an artifact + # allowed to be built by whitelist should be build. + # + # The syntax is the same as for HANDLER_BUILD_WHITELIST, but any matched + # artifact will *not* be rebuild. + # + # HANDLER_BUILD_BLACKLIST = { + # "global": { + # "image": all_( + # {'advisory_name': 'RHSA-.*' + # 'advisory_state: 'SHIPPED_LIVE'}, + # any_( + # {'has_hightouch_bugs': True}, + # {'severity': ['critical', 'important']} + # ) + # ) + # }, + # } + + # ODCS configs + # URL to ODCS to call APIs + ODCS_SERVER_URL = 'https://odcs.localhost/' + ODCS_VERIFY_SSL = True + # Valid authentication method would be kerberos or openidc + ODCS_AUTH_MECH = 'kerberos' + # When use openidc authentcation, set the openidc token for accessing ODCS + ODCS_OPENIDC_TOKEN = '' + + # Kerberos authentication Settings used to authenticated freshmaker itself + # by other services + + # Whether to use keytab to acquire credential cache. keytab should be used + # in a non-devel environment. + KRB_AUTH_USE_KEYTAB = True + # Principal used to acquire credential cache. When using a client keytab, + # this value must be present in that keytab file. Otherwise, principal must + # match the one in specified ccache file. + KRB_AUTH_PRINCIPAL = '' + # Path to freshmaker's client keytab file. + KRB_AUTH_CLIENT_KEYTAB = '' + # Path to credential cache file. This optional could be None when not using + # a client keytab to acquire credential. + KRB_AUTH_CCACHE_FILE = tempfile.mkstemp( + suffix=str(os.getpid()), prefix="freshmaker_cc_") + + # Users are required to be in allowed_clients to generate composes, + # you can add group names or usernames (it can be normal user or host + # principal) into ALLOWED_CLIENTS. The group names are from ldap for + # kerberos users or FAS for openidc users. + ALLOWED_CLIENTS = { + 'groups': [], + 'users': [], + } + + # Users in ADMINS are granted with admin permission. + ADMINS = { + 'groups': [], + 'users': [], + } + + # Select which authentication backend to work with. There are 3 choices + # noauth: no authentication is enabled. Useful for development particularly. + # kerberos: Kerberos authentication is enabled. + # openidc: OpenIDC authentication is enabled. + AUTH_BACKEND = '' + + # Used for Kerberos authentication and to query user's groups. + # Format: ldap://hostname[:port] + # For example: ldap://ldap.example.com/ + AUTH_LDAP_SERVER = '' + + # Group base to query groups from LDAP server. + # Generally, it would be, for example, ou=groups,dc=example,dc=com + AUTH_LDAP_GROUP_BASE = '' + + AUTH_OPENIDC_USERINFO_URI = 'https://id.fedoraproject.org/openidc/UserInfo' + + # OIDC base namespace + # See also section pagure.io/odcs in + # https://fedoraproject.org/wiki/Infrastructure/Authentication + OIDC_BASE_NAMESPACE = 'https://pagure.io/freshmaker/' + + # Scope requested from Fedora Infra for permission of submitting request to + # run a new compose. + # See also: https://fedoraproject.org/wiki/Infrastructure/Authentication + # Add additional required scope in following list + AUTH_OPENIDC_REQUIRED_SCOPES = [ + 'openid', + 'https://id.fedoraproject.org/scope/groups', + ] + + # Select which messaging backend will be used, that could be fedmsg, amq, + # in_memory or rhmsg. + MESSAGING = 'fedmsg' + MESSAGING_BACKENDS = { + 'fedmsg': { + 'SERVICE': 'freshmaker', + }, + 'rhmsg': { + # Brokers to connect, e.g. + # ['amqps://host:5671', 'amqps://anotherhost:5671'] + 'BROKER_URLS': [], + # Path to certificate file used to authenticate freshmaker + 'CERT_FILE': '', + # Path to private key file used to authenticate freshmaker + 'KEY_FILE': '', + # Path to trusted CA certificate bundle. + 'CA_CERT': '', + 'TOPIC_PREFIX': 'VirtualTopic.eng.freshmaker', + }, + 'in_memory': { + 'SERVICE': 'freshmaker', + } + } + + +class DevConfiguration(BaseConfiguration): + DEBUG = True + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.dev', 'org.fedoraproject.stg'] + + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_VERIFY_SSL = False + + # During development, we usually don't need a client keytab to acquire + # credential. Instead, kinit in default ccache with personal principal + # often. + KRB_AUTH_USE_KEYTAB = False + KRB_AUTH_PRINCIPAL = '' # Should be in form name@REAL + # Use the default ccache + KRB_AUTH_CCACHE_FILE = None + + AUTH_BACKEND = 'noauth' + AUTH_OPENIDC_USERINFO_URI = 'https://iddev.fedorainfracloud.org/openidc/UserInfo' + + diff --git a/dev_scripts/errata.py b/dev_scripts/errata.py deleted file mode 100755 index 8ac1a9d..0000000 --- a/dev_scripts/errata.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python -from __future__ import print_function -import os -import sys -from pprint import pprint - -# Allow imports from parent directory. -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -# Set the FRESHMAKER_DEVELOPER_ENV variable. -os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" - -from freshmaker.errata import Errata -from freshmaker import conf - - -if len(sys.argv) != 2: - print("Template for testing freshmaker.Errata class") - print("Usage: ./errata.py ERRATA_TOOL_SERVER_URL") - sys.exit(1) - - -conf.errata_tool_server_url = sys.argv[1] -errata = Errata() - -# Example usage: -data = errata.get_builds(42515) -pprint(data) diff --git a/dev_scripts/lightblue.py b/dev_scripts/lightblue.py deleted file mode 100755 index 4f4ceed..0000000 --- a/dev_scripts/lightblue.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/python -from __future__ import print_function -import os -import sys -from pprint import pprint - -# Allow imports from parent directory. -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -# Set the FRESHMAKER_DEVELOPER_ENV variable. -os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" - -from freshmaker.lightblue import LightBlue - - -if len(sys.argv) != 4: - print("Template for testing freshmaker.Lightblue class") - print("Usage: ./lightblue.py LB_SERVER_URL PATH_TO_LB_CLIENT_CERT " - "PATH_TO_LB_CLIENT_KEY") - sys.exit(1) - - -lb = LightBlue(sys.argv[1], sys.argv[2], sys.argv[3], False) - - -def example_image_request(lb): - query = { - "objectType": "containerImage", - "query": { - "$and": [ - { - "field": "repositories.*.tags.*.name", - "op": "=", - "rvalue": "latest" - }, - { - "field": "brew.build", - "op": "=", - "rvalue": "s2i-core-container-1-66" - }, - { - "field": "parsed_data.files.*.key", - "op": "=", - "rvalue": "buildfile" - } - ] - }, - "projection": lb._get_default_projection(include_rpms=False) - } - images = lb.find_container_images(query) - pprint(images) - - -def example_repository_request(lb): - # Get all the latest container images containing "httpd" - query = { - "objectType": "containerRepository", - "query": { - "$and": [ - { - "field": "images.*.brew.build", - "op": "=", - "rvalue": "s2i-core-container-1-66" - }, - - ] - }, - "projection": [ - {"field": "repository", "include": True}, - ] - } - repositories = lb.find_container_repositories(query) - pprint(repositories) - - -example_image_request(lb) -example_repository_request(lb) diff --git a/dev_scripts/pulp.py b/dev_scripts/pulp.py deleted file mode 100755 index afdf49b..0000000 --- a/dev_scripts/pulp.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/python -from __future__ import print_function -import os -import sys -from pprint import pprint - -# Allow imports from parent directory. -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -# Set the FRESHMAKER_DEVELOPER_ENV variable. -os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" - -from freshmaker.pulp import Pulp - - -if len(sys.argv) != 4: - print("Template for testing freshmaker.Pulp class") - print("Usage: ./pulp.py PULP_SERVER_URL PULP_USERNAME PULP_PASSWORD") - sys.exit(1) - - -pulp = Pulp(sys.argv[1], sys.argv[2], sys.argv[3]) - - -repo_ids = [ - "rhel-7-server-release-e2e-test-1-rpms__x86_64" -] -pprint(pulp.get_content_set_by_repo_ids(repo_ids)) diff --git a/dev_scripts/template_errata.py b/dev_scripts/template_errata.py new file mode 100755 index 0000000..8ac1a9d --- /dev/null +++ b/dev_scripts/template_errata.py @@ -0,0 +1,28 @@ +#!/usr/bin/python +from __future__ import print_function +import os +import sys +from pprint import pprint + +# Allow imports from parent directory. +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +# Set the FRESHMAKER_DEVELOPER_ENV variable. +os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" + +from freshmaker.errata import Errata +from freshmaker import conf + + +if len(sys.argv) != 2: + print("Template for testing freshmaker.Errata class") + print("Usage: ./errata.py ERRATA_TOOL_SERVER_URL") + sys.exit(1) + + +conf.errata_tool_server_url = sys.argv[1] +errata = Errata() + +# Example usage: +data = errata.get_builds(42515) +pprint(data) diff --git a/dev_scripts/template_pulp.py b/dev_scripts/template_pulp.py new file mode 100755 index 0000000..afdf49b --- /dev/null +++ b/dev_scripts/template_pulp.py @@ -0,0 +1,28 @@ +#!/usr/bin/python +from __future__ import print_function +import os +import sys +from pprint import pprint + +# Allow imports from parent directory. +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +# Set the FRESHMAKER_DEVELOPER_ENV variable. +os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" + +from freshmaker.pulp import Pulp + + +if len(sys.argv) != 4: + print("Template for testing freshmaker.Pulp class") + print("Usage: ./pulp.py PULP_SERVER_URL PULP_USERNAME PULP_PASSWORD") + sys.exit(1) + + +pulp = Pulp(sys.argv[1], sys.argv[2], sys.argv[3]) + + +repo_ids = [ + "rhel-7-server-release-e2e-test-1-rpms__x86_64" +] +pprint(pulp.get_content_set_by_repo_ids(repo_ids)) diff --git a/dev_scripts/templates_lightblue.py b/dev_scripts/templates_lightblue.py new file mode 100755 index 0000000..4f4ceed --- /dev/null +++ b/dev_scripts/templates_lightblue.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +from __future__ import print_function +import os +import sys +from pprint import pprint + +# Allow imports from parent directory. +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +# Set the FRESHMAKER_DEVELOPER_ENV variable. +os.environ["FRESHMAKER_DEVELOPER_ENV"] = "1" + +from freshmaker.lightblue import LightBlue + + +if len(sys.argv) != 4: + print("Template for testing freshmaker.Lightblue class") + print("Usage: ./lightblue.py LB_SERVER_URL PATH_TO_LB_CLIENT_CERT " + "PATH_TO_LB_CLIENT_KEY") + sys.exit(1) + + +lb = LightBlue(sys.argv[1], sys.argv[2], sys.argv[3], False) + + +def example_image_request(lb): + query = { + "objectType": "containerImage", + "query": { + "$and": [ + { + "field": "repositories.*.tags.*.name", + "op": "=", + "rvalue": "latest" + }, + { + "field": "brew.build", + "op": "=", + "rvalue": "s2i-core-container-1-66" + }, + { + "field": "parsed_data.files.*.key", + "op": "=", + "rvalue": "buildfile" + } + ] + }, + "projection": lb._get_default_projection(include_rpms=False) + } + images = lb.find_container_images(query) + pprint(images) + + +def example_repository_request(lb): + # Get all the latest container images containing "httpd" + query = { + "objectType": "containerRepository", + "query": { + "$and": [ + { + "field": "images.*.brew.build", + "op": "=", + "rvalue": "s2i-core-container-1-66" + }, + + ] + }, + "projection": [ + {"field": "repository", "include": True}, + ] + } + repositories = lb.find_container_repositories(query) + pprint(repositories) + + +example_image_request(lb) +example_repository_request(lb) diff --git a/docs/dev_scripts.rst b/docs/dev_scripts.rst new file mode 100644 index 0000000..c1b6e55 --- /dev/null +++ b/docs/dev_scripts.rst @@ -0,0 +1,25 @@ +=================== +Development scripts +=================== + +The dev_scripts_ directory contains few useful scripts which can be used to debug Freshmaker locally. + +.. _dev_scripts: https://pagure.io/freshmaker/blob/master/f/dev_scripts + + +General configuration +===================== + +The scripts are configured using the ``config.py`` configuration file. Before you execute any of the development scripts for the first time, you need to create your own ``config.py`` by renaming the ``config.py.template`` and editting the ``EDIT FOLLOWING OPTIONS BASED ON YOUR ENVIRONMENT`` section. + + +``find_images_to_rebuild.py`` +============================= + +This script executes the ``RebuildImagesOnRPMAdvisoryChange`` handler which is responsible for rebuilding container images as a result of RPM RHSA release. This is the main work done by Freshmaker running in the infrastructure. + +Usage: ``./find_images_to_rebuild.py `` + +The script queries Errata Tool, Pulp and Lightblue database to find out the list of container images which would be rebuilt by Freshmaker as a result of this RPM RHSA. It prints the list of container images and also initial set of image builds as submitted to Koji. + +It also creates ``freshmaker.db`` which contains the Event and Artifact builds as they would be created in real infrastructure. diff --git a/docs/index.rst b/docs/index.rst index 660befc..4f2e7ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ compose (mainly the RPM repository) with packages from Koji using the REST API. api_v1 api_v2 messaging_api + dev_scripts Indices and tables ==================