From a2bf91e8682679a7697daa7a80f6a002938e9b01 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: May 19 2017 04:27:45 +0000 Subject: [PATCH 1/2] Add support to query LightBlue * A new class LightBlue to query entities containerImage and containerRepository * New classes ContainerRepository and ContainerImage to represent the repository and image objects * Make it possible to configure LightBlue Signed-off-by: Chenxiong Qi --- diff --git a/conf/config.py b/conf/config.py index 827bf7b..e5af764 100644 --- a/conf/config.py +++ b/conf/config.py @@ -120,6 +120,9 @@ class BaseConfiguration(object): # }, # } + LIGHTBLUE_SERVER_URL = '' # replace with default server url + LIGHTBLUE_VERIFY_SSL = True + class DevConfiguration(BaseConfiguration): DEBUG = True @@ -134,6 +137,8 @@ class DevConfiguration(BaseConfiguration): KOJI_CONTAINER_SCRATCH_BUILD = True + LIGHTBLUE_VERIFY_SSL = False + class TestConfiguration(BaseConfiguration): LOG_BACKEND = 'console' @@ -153,6 +158,9 @@ class TestConfiguration(BaseConfiguration): KOJI_CONTAINER_SCRATCH_BUILD = True + LIGHTBLUE_SERVER_URL = '' # replace with real dev server url + LIGHTBLUE_VERIFY_SSL = False + class ProdConfiguration(BaseConfiguration): pass diff --git a/freshmaker/config.py b/freshmaker/config.py index 1c835d3..0ed824a 100644 --- a/freshmaker/config.py +++ b/freshmaker/config.py @@ -186,6 +186,14 @@ class Config(object): 'default': {}, 'desc': 'Blacklist for build targets of handlers', }, + 'lightblue_server_url': { + 'type': str, + 'default': '', + 'desc': 'Server URL of LightBlue.'}, + 'lightblue_verify_ssl': { + 'type': bool, + 'default': True, + 'desc': 'Whether to enable SSL verification over HTTP with lightblue.'}, } def __init__(self, conf_section_obj): diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py new file mode 100644 index 0000000..fa4287f --- /dev/null +++ b/freshmaker/lightblue.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Written by Chenxiong Qi + +import os +import requests +import json + +from six.moves import http_client + + +class LightBlueRequestFailure(Exception): + """Exception when fail to request from LightBlue""" + + def __init__(self, json_response, status_code): + """Initialize + + :param dict json_response: the JSON data returned from LightBlue + which contains all error information. + :param int status_code: repsonse status code + """ + self._raw = json_response + self._status_code = status_code + + def __repr__(self): + return '<{} [{}]>'.format(self.__class__.__name__, self.status_code) + + def __str__(self): + return 'Error{} ({}):\n{}'.format( + 's' if len(self.errors) > 1 else '', + len(self.errors), + '\n'.join((' {}'.format(err['msg']) for err in self.errors)) + ) + + @property + def raw(self): + return self._raw + + @property + def errors(self): + return self.raw['errors'] + + @property + def status_code(self): + return self._status_code + + +class ContainerRepository(dict): + """Represent a container repository""" + + @classmethod + def create(cls, data): + repo = cls() + repo.update(data) + return repo + + +class ContainerImage(dict): + """Represent a container image""" + + @classmethod + def create(cls, data): + image = cls() + image.update(data) + return image + + +class LightBlue(object): + """Interface to query lightblue""" + + ENTITY_VERSION_CONTAINER_IMAGE = '0.0.12' + ENTITY_VERSION_CONTAINER_REPOSITORY = '0.0.11' + + def __init__(self, server_url, cert, private_key, verify_ssl=None): + self.server_url = server_url + self.api_root = '{}/rest/data/'.format(server_url) + if verify_ssl is None: + self.verify_ssl = True + else: + assert isinstance(verify_ssl, bool) + self.verify_ssl = verify_ssl + + if not os.path.exists(cert): + raise IOError('Certificate file {} does not exist.'.format(cert)) + else: + self.cert = cert + + if not os.path.exists(private_key): + raise IOError('Private key file {} does not exist.'.format(private_key)) + else: + self.private_key = private_key + + def _make_request(self, entity, data): + """Make request to lightblue""" + + entity_url = '{}/{}'.format(self.api_root, entity) + response = requests.post(entity_url, + data=json.dumps(data), + verify=self.verify_ssl, + cert=(self.cert, self.private_key), + headers={'Content-Type': 'application/json'}) + self._raise_expcetion_if_errors_returned(response) + return response.json() + + def _raise_expcetion_if_errors_returned(self, response): + """Raise exception when response contains errors + + :param dict response: the response returned from LightBlue, which is + actually the requests response object. + :raises LightBlueRequestFailure: if response status code is not 200. + Otherwise, just keep silient. + """ + if response.status_code == http_client.OK: + return + raise LightBlueRequestFailure(response.json(), response.status_code) + + def find_container_repositories(self, request): + """Query via entity containerRepository + + :param dict request: a map containing complete query expression. + This query will be sent to LightBlue in a POST request. Refer to + https://jewzaam.gitbooks.io/lightblue-specifications/content/language_specification/query.html + to know more detail about how to write a query. + :return: a list of ContainerRepository objects + :rtype: list + """ + + url = 'find/containerRepository/{}'.format( + self.ENTITY_VERSION_CONTAINER_REPOSITORY) + response = self._make_request(url, request) + + repos = [] + for repo_data in response['processed']: + repo = ContainerRepository() + repo.update(repo_data) + repos.append(repo) + return repos + + def find_container_images(self, request): + """Query via entity containerImage + + :param dict request: a map containing complete query expression. + This query will be sent to LightBlue in a POST request. Refer to + https://jewzaam.gitbooks.io/lightblue-specifications/content/language_specification/query.html + to know more detail about how to write a query. + :return: a list of ContainerImage objects + :rtype: list + """ + + url = 'find/containerImage/{}'.format( + self.ENTITY_VERSION_CONTAINER_IMAGE) + response = self._make_request(url, request) + + images = [] + for image_data in response['processed']: + image = ContainerImage() + image.update(image_data) + images.append(image) + return images diff --git a/requirements.txt b/requirements.txt index 47f8305..83cef9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,4 @@ Flask Flask-Migrate Flask-SQLAlchemy Flask-Script +requests diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py new file mode 100644 index 0000000..f59d972 --- /dev/null +++ b/tests/test_lightblue.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import json +import unittest + +from mock import patch +from six.moves import http_client + +from freshmaker.lightblue import ContainerImage +from freshmaker.lightblue import ContainerRepository +from freshmaker.lightblue import LightBlue +from freshmaker.lightblue import LightBlueRequestFailure + + +class TestLightBlueRequestFailure(unittest.TestCase): + """Test case for exception LightBlueRequestFailure""" + + def setUp(self): + self.fake_error_data = { + 'entity': 'containerImage', + 'entityVersion': '0.0.11', + 'errors': [ + { + 'context': 'rest/FindCommand/containerImage/find(containerImage:0.0.11)/' + 'containerImage/parsed_data/rpm_manifes', + 'errorCode': 'metadata:InvalidFieldReference', + 'msg': 'rpm_manifes in parsed_data.rpm_manifes.*.nvra', + 'objectType': 'error' + } + ], + 'hostname': 'lightbluecrud1.dev2.a1.vary.redhat.com', + 'matchCount': 0, + 'modifiedCount': 0, + 'status': 'ERROR' + } + self.e = LightBlueRequestFailure(self.fake_error_data, + http_client.INTERNAL_SERVER_ERROR) + + def test_get_raw_error_json_data(self): + self.assertEqual(self.fake_error_data, self.e.raw) + + def test_get_status_code(self): + self.assertEqual(http_client.INTERNAL_SERVER_ERROR, self.e.status_code) + + def test_get_inner_errors(self): + self.assertEqual(self.fake_error_data['errors'], self.e.errors) + + def test_errors_listed_in_str(self): + expected_s = '\n'.join((' {}'.format(err['msg']) + for err in self.fake_error_data['errors'])) + self.assertIn(expected_s, str(self.e)) + + +class TestContainerImageObject(unittest.TestCase): + + def test_create(self): + image = ContainerImage.create({ + '_id': '1233829', + 'brew': { + 'completion_date': '20151210T10:09:35.000-0500', + 'build': 'jboss-webserver-3-webserver30-tomcat7-openshift-docker-1.1-6', + 'package': 'jboss-webserver-3-webserver30-tomcat7-openshift-docker' + } + }) + + self.assertEqual('1233829', image['_id']) + self.assertEqual('20151210T10:09:35.000-0500', image['brew']['completion_date']) + + +class TestContainerRepository(unittest.TestCase): + + def test_create(self): + image = ContainerRepository.create({ + 'creationDate': '20160927T11:14:56.420-0400', + 'metrics': { + 'pulls_in_last_30_days': 0, + 'last_update_date': '20170223T08:28:40.913-0500' + } + }) + + self.assertEqual('20160927T11:14:56.420-0400', image['creationDate']) + self.assertEqual(0, image['metrics']['pulls_in_last_30_days']) + self.assertEqual('20170223T08:28:40.913-0500', image['metrics']['last_update_date']) + + +class TestLightBlue(unittest.TestCase): + + def setUp(self): + self.fake_server_url = 'lightblue.localhost' + self.fake_cert_file = 'path/to/cert' + self.fake_private_key = 'path/to/private-key' + + @patch('freshmaker.lightblue.requests.post') + def test_find_container_images(self, post): + post.return_value.status_code = http_client.OK + post.return_value.json.return_value = { + 'modifiedCount': 0, + 'resultMetadata': [], + 'entityVersion': '0.0.12', + 'hostname': self.fake_server_url, + 'matchCount': 2, + 'processed': [ + { + '_id': '57ea8d1f9c624c035f96f4b0', + 'image_id': 'e0f97342ddf6a09972434f98837b5fd8b5bed9390f32f1d63e8a7e4893208af7', + 'brew': { + 'completion_date': '20151210T10:09:35.000-0500', + 'build': 'jboss-webserver-3-webserver30-tomcat7-openshift-docker-1.1-6', + 'package': 'jboss-webserver-3-webserver30-tomcat7-openshift-docker' + }, + }, + { + '_id': '57ea8d289c624c035f96f4db', + 'image_id': 'c1ef3345f36b901b0bddc7ab01ea3f3c83c886faa243e02553f475124eb4b46c', + 'brew': { + 'package': 'sadc-docker', + 'completion_date': '20151203T00:35:30.000-0500', + 'build': 'sadc-docker-7.2-7' + }, + } + ], + 'status': 'COMPLETE', + 'entity': 'containerImage' + } + + fake_request = { + "objectType": "containerImage", + "projection": [ + {"field": "_id", "include": True}, + {"field": "image_id", "include": True}, + {"field": "brew", "include": True, "recursive": True}, + ], + } + + with patch('os.path.exists'): + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + images = lb.find_container_images(request=fake_request) + + post.assert_called_once_with( + '{}/{}/{}'.format(lb.api_root, + 'find/containerImage', + LightBlue.ENTITY_VERSION_CONTAINER_IMAGE), + data=json.dumps(fake_request), + verify=lb.verify_ssl, + cert=(self.fake_cert_file, self.fake_private_key), + headers={'Content-Type': 'application/json'} + ) + self.assertEqual(2, len(images)) + + image = images[0] + self.assertEqual('57ea8d1f9c624c035f96f4b0', image['_id']) + self.assertEqual('jboss-webserver-3-webserver30-tomcat7-openshift-docker', + image['brew']['package']) + + @patch('freshmaker.lightblue.requests.post') + def test_find_container_repositories(self, post): + post.return_value.status_code = http_client.OK + post.return_value.json.return_value = { + 'entity': 'containerRepository', + 'status': 'COMPLETE', + 'modifiedCount': 0, + 'matchCount': 2, + 'processed': [ + { + 'creationDate': '20160927T11:14:56.420-0400', + 'metrics': { + 'pulls_in_last_30_days': 0, + 'last_update_date': '20170223T08:28:40.913-0500' + } + }, + { + 'creationDate': '20161020T04:52:43.365-0400', + 'metrics': { + 'last_update_date': '20170501T03:00:19.892-0400', + 'pulls_in_last_30_days': 20 + } + } + ], + 'entityVersion': '0.0.11', + 'hostname': self.fake_server_url, + 'resultMetadata': [] + } + + fake_request = { + "objectType": "containerRepository", + "projection": [ + {"field": "creationDate", "include": True}, + {"field": "metrics", "include": True, "recursive": True} + ], + } + + with patch('os.path.exists'): + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + repos = lb.find_container_repositories(request=fake_request) + + post.assert_called_once_with( + '{}/{}/{}'.format(lb.api_root, + 'find/containerRepository', + LightBlue.ENTITY_VERSION_CONTAINER_REPOSITORY), + data=json.dumps(fake_request), + verify=lb.verify_ssl, + cert=(self.fake_cert_file, self.fake_private_key), + headers={'Content-Type': 'application/json'} + ) + + self.assertEqual(2, len(repos)) + + repo = repos[0] + self.assertEqual('20160927T11:14:56.420-0400', repo['creationDate']) + self.assertEqual(0, repo['metrics']['pulls_in_last_30_days']) + self.assertEqual('20170223T08:28:40.913-0500', repo['metrics']['last_update_date']) + + @patch('freshmaker.lightblue.requests.post') + def test_raise_error_if_request_data_is_incorrect(self, post): + post.return_value.status_code = http_client.INTERNAL_SERVER_ERROR + post.return_value.json.return_value = { + 'entity': 'containerImage', + 'entityVersion': '0.0.11', + 'errors': [ + { + 'context': 'rest/FindCommand/containerImage/find(containerImage:0.0.11)/' + 'containerImage/parsed_data/rpm_manifes', + 'errorCode': 'metadata:InvalidFieldReference', + 'msg': 'rpm_manifes in parsed_data.rpm_manifes.*.nvra', + 'objectType': 'error' + } + ], + 'hostname': 'lightbluecrud1.dev2.a1.vary.redhat.com', + 'matchCount': 0, + 'modifiedCount': 0, + 'status': 'ERROR' + } + + fake_request = { + "objectType": "containerRepository", + "projection": [ + {"fiel": "creationDate", "include": True}, + ], + } + + with patch('os.path.exists'): + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + self.assertRaises(LightBlueRequestFailure, + lb._make_request, 'find/containerRepository/', fake_request) From b8fa14be4af1e535a80d8cfe835450705c4d17c8 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: May 23 2017 07:07:16 +0000 Subject: [PATCH 2/2] Make entity versions configurable Signed-off-by: Chenxiong Qi --- diff --git a/conf/config.py b/conf/config.py index e5af764..dcdecd5 100644 --- a/conf/config.py +++ b/conf/config.py @@ -123,6 +123,12 @@ class BaseConfiguration(object): LIGHTBLUE_SERVER_URL = '' # replace with default server url LIGHTBLUE_VERIFY_SSL = True + # Lookup versions of each entity: /rest/metadata/{entity name} + LIGHTBLUE_ENTITY_VERSIONS = { + 'containerRepository': '0.0.11', + 'containerImage': '0.0.12', + } + class DevConfiguration(BaseConfiguration): DEBUG = True diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index fa4287f..0c881e4 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -87,10 +87,24 @@ class ContainerImage(dict): class LightBlue(object): """Interface to query lightblue""" - ENTITY_VERSION_CONTAINER_IMAGE = '0.0.12' - ENTITY_VERSION_CONTAINER_REPOSITORY = '0.0.11' - - def __init__(self, server_url, cert, private_key, verify_ssl=None): + def __init__(self, server_url, cert, private_key, + verify_ssl=None, + entity_versions=None): + """Initialize LightBlue instance + + :param str server_url: URL used to call LightBlue APIs. It is + unnecessary to include path part, which will be handled + automatically. For example, https://lightblue.example.com/. + :param str cert: path to certificate file. + :param str private_key: path to private key file. + :param bool verify_ssl: whether to verify SSL over HTTP. Enabled by + default. + :param dict entity_versions: a mapping from entity to what version + should be used to request data. If no such a mapping appear , it + means the default version will be used. You should choose versions + explicitly. If entity_versions is omitted entirely, default version + will be used on each entity. + """ self.server_url = server_url self.api_root = '{}/rest/data/'.format(server_url) if verify_ssl is None: @@ -109,6 +123,20 @@ class LightBlue(object): else: self.private_key = private_key + self.entity_versions = entity_versions or {} + + def _get_entity_version(self, entity_name): + """Lookup configured entity's version + + :param str entity_name: entity name to get its version. + :return: version configured for the entity name. If there is no + corresponding version, emtpy string is returned, which can be used + to construct request URL directly that means to use default + version. + :rtype: str + """ + return self.entity_versions.get(entity_name, '') + def _make_request(self, entity, data): """Make request to lightblue""" @@ -145,7 +173,7 @@ class LightBlue(object): """ url = 'find/containerRepository/{}'.format( - self.ENTITY_VERSION_CONTAINER_REPOSITORY) + self._get_entity_version('entityRespository')) response = self._make_request(url, request) repos = [] @@ -167,7 +195,7 @@ class LightBlue(object): """ url = 'find/containerImage/{}'.format( - self.ENTITY_VERSION_CONTAINER_IMAGE) + self._get_entity_version('containerImage')) response = self._make_request(url, request) images = [] diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index f59d972..b60bc67 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -23,6 +23,7 @@ import json import unittest +from mock import call from mock import patch from six.moves import http_client @@ -103,7 +104,7 @@ class TestContainerRepository(unittest.TestCase): self.assertEqual('20170223T08:28:40.913-0500', image['metrics']['last_update_date']) -class TestLightBlue(unittest.TestCase): +class TestQueryEntityFromLightBlue(unittest.TestCase): def setUp(self): self.fake_server_url = 'lightblue.localhost' @@ -159,9 +160,7 @@ class TestLightBlue(unittest.TestCase): images = lb.find_container_images(request=fake_request) post.assert_called_once_with( - '{}/{}/{}'.format(lb.api_root, - 'find/containerImage', - LightBlue.ENTITY_VERSION_CONTAINER_IMAGE), + '{}/{}/'.format(lb.api_root, 'find/containerImage'), data=json.dumps(fake_request), verify=lb.verify_ssl, cert=(self.fake_cert_file, self.fake_private_key), @@ -218,9 +217,7 @@ class TestLightBlue(unittest.TestCase): repos = lb.find_container_repositories(request=fake_request) post.assert_called_once_with( - '{}/{}/{}'.format(lb.api_root, - 'find/containerRepository', - LightBlue.ENTITY_VERSION_CONTAINER_REPOSITORY), + '{}/{}/'.format(lb.api_root, 'find/containerRepository'), data=json.dumps(fake_request), verify=lb.verify_ssl, cert=(self.fake_cert_file, self.fake_private_key), @@ -268,3 +265,65 @@ class TestLightBlue(unittest.TestCase): private_key=self.fake_private_key) self.assertRaises(LightBlueRequestFailure, lb._make_request, 'find/containerRepository/', fake_request) + + +class TestEntityVersion(unittest.TestCase): + """Test case for ensuring correct entity version in request""" + + def setUp(self): + self.fake_server_url = 'lightblue.localhost' + self.fake_cert_file = 'path/to/cert' + self.fake_private_key = 'path/to/private-key' + self.fake_entity_versions = { + 'containerImage': '0.0.11' + } + + @patch('freshmaker.lightblue.LightBlue._make_request') + @patch('os.path.exists') + def test_use_default_entity_version(self, exists, _make_request): + exists.return_value = True + + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key, + entity_versions=self.fake_entity_versions) + fake_request = {} + lb.find_container_repositories({}) + + _make_request.assert_called_once_with('find/containerRepository/', {}) + + @patch('freshmaker.lightblue.LightBlue._make_request') + @patch('os.path.exists') + def test_use_specified_entity_version(self, exists, _make_request): + exists.return_value = True + + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key, + entity_versions=self.fake_entity_versions) + fake_request = {} + lb.find_container_images({}) + + _make_request.assert_called_once_with('find/containerImage/0.0.11', {}) + + @patch('freshmaker.lightblue.LightBlue._make_request') + @patch('os.path.exists') + def test_use_default_entity_version_when_parameter_is_omitted( + self, exists, _make_request): + exists.return_value = True + _make_request.return_value = { + # Omit other attributes that are not useful for this test + 'processed': [] + } + + lb = LightBlue(server_url=self.fake_server_url, + cert=self.fake_cert_file, + private_key=self.fake_private_key) + fake_request = {} + lb.find_container_repositories({}) + lb.find_container_images({}) + + _make_request.assert_has_calls([ + call('find/containerRepository/', {}), + call('find/containerImage/', {}), + ])