From 370532e16fc69497b7070ddf386f7da8a338624c Mon Sep 17 00:00:00 2001 From: Giulia Naponiello Date: Jun 08 2020 14:31:24 +0000 Subject: Get content_sets for all architectures When setting generate_pulp_repos to True, we should ensure that the content_sets for the one image contains the content sets for all the architectures of that image. Let's do this for every container image returned from LightBlue, and not that late in the process instead. ref: CLOUDWF-1561 Signed-off-by: Giulia Naponiello --- diff --git a/freshmaker/lightblue.py b/freshmaker/lightblue.py index 10850ee..512578d 100644 --- a/freshmaker/lightblue.py +++ b/freshmaker/lightblue.py @@ -594,6 +594,26 @@ class LightBlue(object): arch_image.update_multi_arch(image) image.update_multi_arch(arch_image) + # There can be multi-arch images which share the same + # image['brew']['build']. Freshmaker is not interested in the image + # architecture, it is only interested in NVR, so group the images + # by the same image['brew']['build'] and include just first one in the + # image list. + sorted_images = sorted_by_nvr(images, reverse=True) + images = [] + + # We must combine content_sets with same image NVR + # but different architectures into one content_sets field + for k, temp_images in groupby(sorted_images, key=lambda item: item.nvr): + temp_images = list(temp_images) + img = temp_images[0] + if 'content_sets' in img and len(temp_images) > 1: + new_content_sets = set(img.get('content_sets')) + for i in temp_images[1:]: + new_content_sets.update(i.get('content_sets', [])) + img["content_sets"] = list(new_content_sets) + images.append(img) + return images def _set_container_repository_filters( @@ -1161,25 +1181,6 @@ class LightBlue(object): # therefore set `published` to None. images = self.get_images_by_nvrs( leaf_container_images, None, content_sets, srpm_nvrs) - # There can be multi-arch images which share the same - # image['brew']['build']. Freshmaker is not interested in the image - # architecture, it is only interested in NVR, so group the images - # by the same image['brew']['build'] and include just first one in the - # image list. - sorted_images = sorted_by_nvr(images, reverse=True) - images = [] - - # We must combine content_sets with same image NVR - # but different architectures into one content_sets field - for k, temp_images in groupby(sorted_images, key=lambda item: item.nvr): - temp_images = list(temp_images) - img = temp_images[0] - if 'content_sets' in img and len(temp_images) > 1: - new_content_sets = set(img.get('content_sets')) - for i in temp_images[1:]: - new_content_sets.update(i.get('content_sets', [])) - img["content_sets"] = list(new_content_sets) - images.append(img) # In case we query for unpublished images, we need to return just # the latest NVR for given name-version, otherwise images would diff --git a/tests/test_lightblue.py b/tests/test_lightblue.py index ec91c47..6bba125 100644 --- a/tests/test_lightblue.py +++ b/tests/test_lightblue.py @@ -834,6 +834,10 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): self.assertEqual(2, len(images)) image = images[0] + self.assertEqual('57ea8d289c624c035f96f4db', image['_id']) + self.assertEqual('sadc-docker', + image['brew']['package']) + image = images[1] self.assertEqual('57ea8d1f9c624c035f96f4b0', image['_id']) self.assertEqual('jboss-webserver-3-webserver30-tomcat7-openshift-docker', image['brew']['package']) @@ -858,6 +862,7 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): 'build': 'sadc-container-1.1-6', 'package': 'sadc-container', }, + 'content_sets': ['dummy-content-set-1'], }, { '_id': '57ea8d289c624c035f96f4db', @@ -868,6 +873,7 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): 'build': 'sadc-container-1.1-6', 'package': 'sadc-container', }, + 'content_sets': ['dummy-content-set-2'], } ], 'status': 'COMPLETE', @@ -897,7 +903,7 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): cert=(self.fake_cert_file, self.fake_private_key), headers={'Content-Type': 'application/json'} ) - self.assertEqual(2, len(images)) + self.assertEqual(1, len(images)) # Verify update_multi_arch is first called with the second image, # then with the first image. This is to ensure all ContainerImage # objects for the same Brew build have the same multi arch data. @@ -2032,9 +2038,10 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): new_images = [ContainerImage.create(i) for i in new_images] find_repos.return_value = self.fake_repositories_with_content_sets find_images.return_value = self.fake_container_images + new_images - right_content_sets = [["dummy-content-set-1"], - ["dummy-content-set-1", "dummy-content-set-2"], - ["content-set-1", "content-set-2", "content-set-3"]] + right_content_sets = [["dummy-content-set-1", "dummy-content-set-2"], + ["dummy-content-set-1"], + ["content-set-1", "content-set-2"], + ["content-set-2", "content-set-3"]] with patch('os.path.exists'): lb = LightBlue(server_url=self.fake_server_url, cert=self.fake_cert_file, @@ -2044,7 +2051,7 @@ class TestQueryEntityFromLightBlue(helpers.FreshmakerTestCase): ["content-set-1", "content-set-2", "content-set-3"], leaf_container_images=['placeholder']) - self.assertEqual(3, len(ret)) + self.assertEqual(4, len(ret)) images_content_sets = [sorted(i.get('content_sets', ['!'])) for i in ret] self.assertEqual(images_content_sets, right_content_sets)