From c1809f86567191ae91778160f7b3fb358cc47262 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 14 2017 11:29:54 +0000 Subject: [PATCH 1/8] Preserve quoted substrings in ENV variables --- diff --git a/modularity/oc_template.py b/modularity/oc_template.py index 85ee975..b3f472d 100644 --- a/modularity/oc_template.py +++ b/modularity/oc_template.py @@ -6,6 +6,7 @@ import yaml import tempfile import shutil import re +import shlex from dockerfile_parse import DockerfileParser @@ -85,7 +86,7 @@ class OpenShiftTemplateGenerator(object): def _get_env(self, value): """Function gets env as field""" - return value.split(" ") + return shlex.split(value) def _get_volume(self, value): """Function evaluates a value and returns as string.""" From 2da59ad452589f3c66686b7c74427a628d18061a Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 14 2017 12:10:13 +0000 Subject: [PATCH 2/8] Fixup - New format of maintainer label --- diff --git a/modularity/oc_template.py b/modularity/oc_template.py index b3f472d..10b803e 100644 --- a/modularity/oc_template.py +++ b/modularity/oc_template.py @@ -104,7 +104,14 @@ class OpenShiftTemplateGenerator(object): return None labels = re.sub('\s\s+', ';', value).split(';') labels = [l.replace('"', '') for l in labels] - label_dict = {l.split('=')[0]: l.split('=')[1] for l in labels} + label_dict = {} + for l in labels: + if len(l.split('=')) == 2: + label_dict[l.split('=')[0]] = l.split('=')[1] + elif re.match('maintainer', l, re.I): + label_dict['maintainer'] = l.split(' ',1)[1] + else: + raise ValueError("Unrecogised label: ", l) return label_dict def _get_docker_tags(self): From 149ba77577c4d4fdb0d8f3bcb5873102c083b0d3 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 19 2017 12:38:16 +0000 Subject: [PATCH 3/8] Centralise scripts into modtools --- diff --git a/contrib/__init__.py b/contrib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/contrib/__init__.py diff --git a/contrib/modtools.py b/contrib/modtools.py new file mode 100755 index 0000000..b4418f8 --- /dev/null +++ b/contrib/modtools.py @@ -0,0 +1,74 @@ +#!/usr/bin/python2 + +import sys +import argparse + +from modularity.module_generator import ModuleGenerator +from modularity.oc_template import OpenShiftTemplateGenerator + + +class ModtoolsCLI(object): + """ Class for processing data from commandline """ + + @staticmethod + def build_parser(): + parser = argparse.ArgumentParser(description="Generates module related files") + subparsers = parser.add_subparsers(dest="cmd_name") + + parser_rpm2module = subparsers.add_parser( + 'rpm2module', help="Generates modulemd file", + description="Gets package info and dependencies and creates modulemd file." + ) + parser_rpm2module.add_argument( + "pkgs", + metavar='PKGS', + nargs='+', + help="Specify list of packages for module.", + ) + + parser_docker2openshift = subparsers.add_parser( + "docker2openshift", help="Generates openshift template from dockerfile", + description="Creates an OpenShift template YAML file.", + ) + + parser_docker2openshift.add_argument( + "image", + metavar='IMAGE', + help="docker image name (like NAME or docker.io/USER/NAME)", + ) + parser_docker2openshift.add_argument( + "--dockerfile", + help="Specify Dockerfile name. Default is Dockerfile." + ) + + return parser + + def __init__(self, args=None): + self.parser = ModtoolsCLI.build_parser() + self.args = self.parser.parse_args(args) + + def __getattr__(self, name): + try: + return getattr(self.args, name) + except AttributeError: + return object.__getattribute__(self, name) + + +class ModtoolsCLIHelper(object): + + @staticmethod + def run(): + try: + cli = ModtoolsCLI(sys.argv[1:]) + if cli.args.cmd_name == 'rpm2module': + mg = ModuleGenerator(cli.args.pkgs) + mg.run() + + if cli.args.cmd_name == 'docker2openshift': + otg = OpenShiftTemplateGenerator(cli.args) + otg.run() + except KeyboardInterrupt: + print('\nInterrupted by user') + except Exception as e: + print(e) + sys.exit(1) \ No newline at end of file diff --git a/diff_module_deps.py b/diff_module_deps.py deleted file mode 100755 index fc48aea..0000000 --- a/diff_module_deps.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 - -import sys -from modularity.cli import CliModuleDepsDifferHelper - -if __name__ == "__main__": - sys.exit(CliModuleDepsDifferHelper.run()) diff --git a/docker2openshift.py b/docker2openshift.py deleted file mode 100755 index 899ff44..0000000 --- a/docker2openshift.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python - -import sys - -from modularity.cli import CliHelper - -if __name__ == "__main__": - sys.exit(CliHelper.run()) diff --git a/modtools b/modtools new file mode 100755 index 0000000..867d17c --- /dev/null +++ b/modtools @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sys +from contrib.modtools import ModtoolsCLIHelper + + +if __name__ == "__main__": + cli = ModtoolsCLIHelper() + sys.exit(cli.run()) diff --git a/modularity/cli.py b/modularity/cli.py deleted file mode 100644 index b4213d6..0000000 --- a/modularity/cli.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -# - -import sys -import argparse -from modularity.module_generator import ModuleGenerator - -from modularity.oc_template import OpenShiftTemplateGenerator -from .module_deps_differ import ModuleDepsDiffer - - -class CLI(object): - """ Class for processing data from commandline """ - - @staticmethod - def build_parser(): - parser = argparse.ArgumentParser(description="Creates an OpenShift template YAML file.") - parser.add_argument( - "image", - metavar='IMAGE', - help="docker image name (like NAME or docker.io/USER/NAME)", - ) - parser.add_argument( - "--dockerfile", - help="Specify Dockerfile name. Default is Dockerfile." - ) - return parser - - def __init__(self, args=None): - self.parser = CLI.build_parser() - self.args = self.parser.parse_args(args) - - def __getattr__(self, name): - try: - return getattr(self.args, name) - except AttributeError: - return object.__getattribute__(self, name) - - -class CliHelper(object): - - @staticmethod - def run(): - try: - cli = CLI(sys.argv[1:]) - otg = OpenShiftTemplateGenerator(cli) - otg.run() - except KeyboardInterrupt: - print('\nInterrupted by user') - # except Exception as e: - # print(e) - # sys.exit(1) - - -class CLIRpm2Module(object): - """ Class for processing data from commandline """ - - @staticmethod - def build_parser(): - parser = argparse.ArgumentParser(description="Creates an modulesMD file.") - parser.add_argument( - "pkgs", - metavar='PKGS', - nargs='+', - help="Specify list of packages for module.", - ) - return parser - - def __init__(self, args=None): - self.parser = CLIRpm2Module.build_parser() - self.args = self.parser.parse_args(args) - - def __getattr__(self, name): - try: - return getattr(self.args, name) - except AttributeError: - return object.__getattribute__(self, name) - - -class CliModuleDepsDiffer(object): - - def __init__(self, args=None): - self.parser = CliModuleDepsDiffer.build_parser() - self.args = self.parser.parse_args(args) - - @staticmethod - def build_parser(): - parser = argparse.ArgumentParser(description='k;') - parser.add_argument( - "pkgs", - metavar='PKGS', - nargs='+', - help="Names of packages for dependency resolution" - ) - parser.add_argument( - "--resolve", - action='store_true', - default=False - ) - - return parser - - def __getattr__(self, name): - try: - return getattr(self.args, name) - except AttributeError: - return object.__getattribute__(self, name) - - -class CliModuleDepsDifferHelper(object): - @staticmethod - def run(): - try: - cli = CliModuleDepsDiffer(sys.argv[1:]) - mdd = ModuleDepsDiffer(cli.args.pkgs) - mdd.run() - if cli.resolve: - mdd.dump_pkgs_result() - else: - mdd.dump_caps_result() - except KeyboardInterrupt: - print('\nInterrupted by user') - - -class CliRpm2ModuleHelper(object): - - @staticmethod - def run(): - try: - cli = CLIRpm2Module(sys.argv[1:]) - mg = ModuleGenerator(cli.args.pkgs) - mg.run() - - except KeyboardInterrupt: - print('\nInterrupted by user') - # except Exception as e: - # print(e) - # sys.exit(1) diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index a6e32e6..f5b7e3e 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import os import sys diff --git a/modularity/module_generator.py b/modularity/module_generator.py index d23fdf4..9781687 100644 --- a/modularity/module_generator.py +++ b/modularity/module_generator.py @@ -36,6 +36,7 @@ class ModuleGenerator(object): are information taken from SPEC file. :return: """ + self.mmd.add_module_license("MIT") if len(self.pkgs) == 1: self.mmd.summary = str(self.pkg.summary) @@ -43,12 +44,12 @@ class ModuleGenerator(object): # Default license for the module metadata, same as default Fedora # content license. - self.mmd.add_module_license("MIT") self.mmd.add_content_license(str(self.pkg.license)) for pkg in ModuleDepsDiffer.whatprovides(self.pkgs): self.mmd.api.add_rpm(pkg) + self.mmd.components.add_rpm(pkg, "Package in api", buildorder=self._get_build_order(pkg)) for pkg in (self.build_deps - self.mmd.api.rpms - self.run_deps): self.mmd.filter.add_rpm(pkg) @@ -66,7 +67,6 @@ class ModuleGenerator(object): if mod == 'result': continue if caps: - # TODO: use specific stream? name, stream = self.differ.get_module_name(mod).split(':') self.mmd.add_buildrequires(name, stream) diff --git a/rpm2module.py b/rpm2module.py deleted file mode 100755 index d5712ef..0000000 --- a/rpm2module.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python - -import sys -from modularity.cli import CliRpm2ModuleHelper - -if __name__ == "__main__": - sys.exit(CliRpm2ModuleHelper.run()) diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index 9cce04f..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from modularity.cli import CLI - - -class TestCLI(object): - """ - The test suite is used for testing CLI class - """ - def test_cli_unit(self): - """Function tests cli class with all arguments""" - conf = {'dockerfile': 'Dockerfile.testing', - 'image': 'Testing_image'} - arguments = ['--dockerfile', 'Dockerfile.testing', - 'Testing_image'] - cli = CLI(arguments) - for key, value in cli.args.__dict__.items(): - assert cli.args.__dict__[key] == conf[key] diff --git a/tests/test_oc_template.py b/tests/test_oc_template.py index 775e316..e5b7d70 100644 --- a/tests/test_oc_template.py +++ b/tests/test_oc_template.py @@ -8,14 +8,14 @@ import os import six import urllib -from modularity.cli import CLI +from contrib.modtools import ModtoolsCLI from modularity.oc_template import OpenShiftTemplateGenerator from modularity.oc_template import VOLUME, ENV, EXPOSE, LABEL def init_oc_template_generator(dockerfile, image_name, working_dir): - arguments = ['--dockerfile', dockerfile, image_name] - cli = CLI(arguments) + arguments = ['docker2openshift','--dockerfile', dockerfile, image_name] + cli = ModtoolsCLI(arguments) ostg = OpenShiftTemplateGenerator(cli, working_dir) oc_template = urllib.URLopener() oc_template.retrieve('https://raw.githubusercontent.com/container-images/container-image-template/master/openshift-template.yml', @@ -258,7 +258,6 @@ class TestOCTemplate(object): assert tmpl == expected_tmpl - class TestCockpitDockerFromDistGit(object): ostg = None WORKING_DIR = '' @@ -270,7 +269,7 @@ class TestCockpitDockerFromDistGit(object): self.WORKING_DIR = tempfile.mkdtemp(prefix="ostg-") self.cockpit_name = 'Dockerfile-Cockpit' docker = urllib.URLopener() - docker.retrieve('http://pkgs.fedoraproject.org/cgit/docker/cockpit.git/plain/Dockerfile', + docker.retrieve('http://pkgs.fedoraproject.org/cgit/container/cockpit.git/plain/Dockerfile', os.path.join(os.path.dirname(__file__), self.cockpit_name)) self.ostg = init_oc_template_generator(self.cockpit_name, 'docker_image', From e69bbc190597c80eca95719e57840e69fd2d6750 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 20 2017 10:53:29 +0000 Subject: [PATCH 4/8] Add tests for module generator --- diff --git a/contrib/modtools.py b/contrib/modtools.py index b4418f8..2831a8b 100755 --- a/contrib/modtools.py +++ b/contrib/modtools.py @@ -71,4 +71,4 @@ class ModtoolsCLIHelper(object): print('\nInterrupted by user') except Exception as e: print(e) - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/tests/test_module_generator.py b/tests/test_module_generator.py new file mode 100644 index 0000000..7c1056c --- /dev/null +++ b/tests/test_module_generator.py @@ -0,0 +1,68 @@ +import pytest +import os.path +from contrib.modtools import ModtoolsCLI +from modularity.module_generator import ModuleGenerator + + +class TestSinglePackageInput(object): + + def setup(self): + self.input = 'grep' + # TODO: test this on modtools when it is packaged + cmd_input = list(['rpm2module']) + cmd_input.append(self.input) + cli = ModtoolsCLI(cmd_input) + mg = ModuleGenerator(cli.pkgs) + self.md = mg + mg.run() + + def teardown(self): + os.remove(self.input + '.yaml') + + def test_description(self): + assert (self.md.mmd.description == 'The GNU versions of commonly used grep utilities. Grep searches through\n' + + 'textual input for lines which contain a match to a specified pattern and then\n' + + 'prints the matching lines. GNU\'s grep utilities include grep, egrep and fgrep.\n\n' + + 'GNU grep is needed by many scripts, so it shall be installed on every system.') + + def test_licences(self): + assert(len(self.md.mmd.module_licenses) == 1) + assert(sorted(self.md.mmd.module_licenses) == sorted(['MIT'])) + assert(len(self.md.mmd.content_licenses) == 1) + assert(sorted(self.md.mmd.content_licenses) == sorted(['GPLv3+'])) + + def test_summary(self): + assert(self.md.mmd.summary == "Pattern matching utilities") + + def test_api(self): + assert(len(self.md.mmd.api.rpms) == 1) + assert(sorted(self.md.mmd.api.rpms) == sorted([self.input])) + + def test_output(self): + assert(os.path.isfile(self.input + '.yaml')) + + +class TestMultiplePackageInput(object): + + def setup(self): + self.input = ['grep','mariadb'] + cmd_input = list(['rpm2module']) + cmd_input.extend(self.input) + cli = ModtoolsCLI(cmd_input) + mg = ModuleGenerator(cli.pkgs) + self.md = mg + mg.run() + + def teardown(self): + os.remove('modulemd-output.yaml') + + def test_licences(self): + assert(len(self.md.mmd.module_licenses) == 1) + assert(sorted(self.md.mmd.module_licenses) == sorted(['MIT'])) + + def test_api(self): + assert(len(self.md.mmd.api.rpms) == len(self.input)) + assert(sorted(self.md.mmd.api.rpms) == sorted(self.input)) + + def test_output(self): + assert (os.path.isfile('modulemd-output.yaml')) From 294c8d524e3ff7b8ac61c979d41961d419f770bd Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 20 2017 10:53:47 +0000 Subject: [PATCH 5/8] Update documentation --- diff --git a/README.md b/README.md index 1706fc5..8d16127 100644 --- a/README.md +++ b/README.md @@ -4,29 +4,29 @@ Following packages are required: * python2-pytest -## docker2openshift.py +## modtools docker2openshift The tool is used for generation OpenShift template from Dockerfile and from [https://github.com/container-images/container-image-template/blob/master/openshift-template.yml](openshift-template.yml) already prepared by Modularity team. As Dockerfile as openshift-template.yml have to exist in the directory. -### How to use docker2openshift.py +### How to use modtools docker2openshift Run the command with following parameters: - `./docker2openshift.py --dockerfile IMAGENAME` + `./modtools docker2openshift --dockerfile IMAGENAME` where parameters mean: * dockerfile ... means full path to Dockerfile * IMAGENAME ... means image name in register. Can be taken from command `docker images` or from some other registry. -The output of **docker2openshift.py** command is OpenShift template stored in temporary directory +The output of **modtools docker2openshift** command is OpenShift template stored in temporary directory Example usage: ```bash - ./docker2openshift.py --dockerfile ~/work/programming/container-images/debugging-tools/Dockerfile registry.fedoraproject.org/debugging-tools + ./modtools docker2openshift --dockerfile ~/work/programming/container-images/debugging-tools/Dockerfile registry.fedoraproject.org/debugging-tools /home/phracek/work/programming/container-images/debugging-tools/Dockerfile OpenShift template is generated here: /tmp/tmpM0teUC/openshift-template.yml ``` -### How to run tests for docker2openshift.py +### How to run tests In order to run tests, run command: @@ -34,7 +34,7 @@ In order to run tests, run command: ### Modulemd creation -./rpm2modulemd.py script creates modulemd file from package names. Output is written in modulemd-output.yaml +modtools rpm2module script creates modulemd file from package names. Output is written in modulemd-output.yaml file (multiple packages as input) or in file named after package name (single package as input). Please make sure you have latest module-build-service and dnf installed before running. From ec6d4f32b7ae89107443d14936ed01fe73335a09 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 20 2017 10:53:50 +0000 Subject: [PATCH 6/8] Init files for rpm build --- diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..eebc271 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include MANIFEST.in +include README.md +include modtools +recursive-include modularity * diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..680b7fe --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +try: + from setuptools import setup, find_packages +except ImportError: + from distutils.core import setup + +setup( + name='modtools', + version='0.1', + author='Dominika Hodovska', + author_email='dhodovsk@redhat.com', + description='Utilities for creating and managing modules', + long_description='Modtools now provides tools generating openshift templates from module Dockerfiles' + 'and creating modulemd files from package names (intended api of module).', + license='MIT', + keywords='modularization modularity module modulemd openshift template docker', + url='https://pagure.io/modularity/modularity-tools', + scripts=['modtools'], + packages=find_packages(), +) From df7df3c344c5b88db07e18edf2392a52676ce30d Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 23 2017 09:53:00 +0000 Subject: [PATCH 7/8] Set python3 as default --- diff --git a/contrib/modtools.py b/contrib/modtools.py index 2831a8b..5b6f6fc 100755 --- a/contrib/modtools.py +++ b/contrib/modtools.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 import sys import argparse diff --git a/modtools b/modtools index 867d17c..9bfb6f7 100755 --- a/modtools +++ b/modtools @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys from contrib.modtools import ModtoolsCLIHelper diff --git a/modularity/module_deps_differ.py b/modularity/module_deps_differ.py index f5b7e3e..ed396fd 100755 --- a/modularity/module_deps_differ.py +++ b/modularity/module_deps_differ.py @@ -31,7 +31,7 @@ def get_pkgs_source_rpm_name(pkg): source_rpm_name = source_rpm.rsplit('-', 2)[0] else: source_rpm_name = pkg.source_name - return source_rpm_name.encode('ascii', 'ignore') + return source_rpm_name def get_pdc_client(): @@ -128,7 +128,7 @@ class ModuleDepsDiffer(object): source_pkgs = [] base = ModuleDepsDiffer.get_base_from_repo('f26-source', REPO_F26_SOURCE) for pkg in pkgs: - source_rpm_name = get_pkgs_source_rpm_name(pkgs[0]) + source_rpm_name = get_pkgs_source_rpm_name(pkg) filter_result = base.sack.query().filter(name=source_rpm_name) ModuleDepsDiffer.package_unique(filter_result) source_pkgs.append(filter_result[0]) From 8642f6da19e4213c63e583b55d86ed4673c867d0 Mon Sep 17 00:00:00 2001 From: Dominika Hodovska Date: Jun 23 2017 10:00:14 +0000 Subject: [PATCH 8/8] Fix documentation and version --- diff --git a/README.md b/README.md index 8d16127..2f7413b 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ The output of **modtools docker2openshift** command is OpenShift template stored Example usage: ```bash - ./modtools docker2openshift --dockerfile ~/work/programming/container-images/debugging-tools/Dockerfile registry.fedoraproject.org/debugging-tools -/home/phracek/work/programming/container-images/debugging-tools/Dockerfile + ./modtools docker2openshift --dockerfile path/to/Dockerfile image/repository/url +/path/to/Dockerfile OpenShift template is generated here: /tmp/tmpM0teUC/openshift-template.yml ``` ### How to run tests diff --git a/setup.py b/setup.py index 680b7fe..f5fa086 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ except ImportError: setup( name='modtools', - version='0.1', + version='0.0.1', author='Dominika Hodovska', author_email='dhodovsk@redhat.com', description='Utilities for creating and managing modules',