From cf20a2e0eb5299d3a8031981d0eedf193bd161ef Mon Sep 17 00:00:00 2001 From: Yaakov Selkowitz Date: Nov 10 2023 18:00:57 +0000 Subject: Add init subcommand This is based on and replaces fedmod rpm2flatpak, as we have moved away from modularity in flatpak builds from F39. --- diff --git a/flatpak_module_tools/cli.py b/flatpak_module_tools/cli.py index 23fe3d9..354fe66 100644 --- a/flatpak_module_tools/cli.py +++ b/flatpak_module_tools/cli.py @@ -18,6 +18,7 @@ from .console_logging import ConsoleHandler from .flatpak_builder import (FLATPAK_METADATA_ANNOTATIONS, FLATPAK_METADATA_BOTH, FLATPAK_METADATA_LABELS) +from .flatpak_generator import FlatpakGenerator from .koji_utils import watch_koji_task from .installer import Installer from .rpm_builder import RpmBuilder @@ -484,3 +485,31 @@ def build_rpms_local( builder.build_rpms_local( manual_packages, manual_repos, auto=auto, allow_outdated=allow_outdated ) + + +@cli.command() +@click.option("--flathub", metavar="ID_OR_SEARCH_TERM", + help="Initialize from a Flathub Flatpak.") +@click.option("--runtime-name", metavar="RUNTIME", + help="Specify runtime-name, defaults to 'flatpak-runtime'") +@click.option("--runtime-version", metavar="VERSION", + help="Specify runtime-version, defaults to latest stable release") +@click.option("--output-containerspec", metavar="FILE", + help="Write container specification to FILE" + " instead of container.yaml.") +@click.option("--force", "-f", is_flag=True, + help="Overwriting existing output files") +@click.argument("package", metavar='PACKAGE', required=True) +@click.pass_context +def init( + ctx, + output_containerspec: Optional[Path], + flathub: Optional[str], + force: bool, + runtime_name: Optional[str], + runtime_version: Optional[str], + package: List[str] +): + """Generate container.yaml from an RPM""" + fg = FlatpakGenerator(package) + fg.run(output_containerspec, force=force, flathub=flathub, runtime_name=runtime_name, runtime_version=runtime_version) diff --git a/flatpak_module_tools/flatpak_generator.py b/flatpak_module_tools/flatpak_generator.py new file mode 100644 index 0000000..e90b712 --- /dev/null +++ b/flatpak_module_tools/flatpak_generator.py @@ -0,0 +1,193 @@ +import json +import os +import re +import sys +from textwrap import dedent + +import click +import requests +import yaml + + +# Some PyYAML magic to get the output we want for container.yaml + +class LiteralScalar(str): + """String subclass that gets dumped into yaml as a scalar""" + pass + + +def _represent_literal_scalar(dumper, s): + return dumper.represent_scalar(tag=u'tag:yaml.org,2002:str', + value=s, + style='|') + + +yaml.add_representer(LiteralScalar, _represent_literal_scalar) + + +class NoSortMapping(dict): + """dict subclass that dumped into yaml as a scalar without sorting keys""" + pass + + +def _represent_no_sort_mapping(dumper, d): + return yaml.MappingNode(tag='tag:yaml.org,2002:map', + value=[(dumper.represent_data(k), + dumper.represent_data(v)) + for k, v in d.items()], + flow_style=False) + + +yaml.add_representer(NoSortMapping, _represent_no_sort_mapping) + + +def _load_flathub_manifest(search_term): + response = requests.get("https://flathub.org/api/v1/apps") + response.raise_for_status() + apps = response.json() + + matches = [] + search_lower = search_term.lower() + for app in apps: + if (search_lower in app['flatpakAppId'].lower() or + search_lower in app['name'].lower()): + matches.append((app['flatpakAppId'], app['name'])) + + if len(matches) > 1: + max_id_len = max([len(app_id) for app_id, _ in matches]) + for app_id, name in matches: + print(app_id + (' ' * (max_id_len - len(app_id)) + ' ' + name)) + raise click.ClickException("Multiple matches found on flathub.org") + elif len(matches) == 0: + raise click.ClickException("No match found on flathub.org") + + app_id = matches[0][0] + + for fname, is_yaml in [ + (f"{app_id}.json", False), + (f"{app_id}.yaml", True), + (f"{app_id}.yml", True)]: + url = f"https://raw.githubusercontent.com/flathub/{app_id}/master/{fname}" + response = requests.get(url) + if response.status_code == 404: + continue + else: + break + + response.raise_for_status() + + if is_yaml: + return yaml.safe_load(response.text) + else: + # flatpak-builder supports non-standard comments in the manifest, strip + # them out. (Ignore the possibility of C comments embedded in strings.) + # + # Regex explanation: matches /**/ (multiline) + # DOES NOT contains "/*" substring + no_comments = re.sub(r'/\*((?!/\*).)*?\*/', '', response.text, flags=re.DOTALL) + return json.loads(no_comments) + + +class FlatpakGenerator(str): + def __init__(self, pkg): + self.pkg = pkg + + def _flathub_container_yaml(self, manifest, runtime_name, runtime_version): + app_id = manifest.get('app-id') + if app_id is None: + app_id = manifest['id'] + yml = NoSortMapping({ + 'flatpak': NoSortMapping({ + 'id': app_id, + 'branch': 'stable', + 'runtime-name': runtime_name, + 'runtime-version': 'f' + str(runtime_version), + }) + }) + + yml['flatpak']['packages'] = [self.pkg] + + for key in ['command', + 'appstream-license', + 'appstream-compose', + 'desktop-file-name-prefix', + 'desktop-file-name-suffix', + 'rename-appdata-file', + 'rename-desktop-file', + 'rename-icon', + 'copy-icon']: + if key in manifest: + yml['flatpak'][key] = manifest[key] + + if 'finish-args' in manifest: + yml['flatpak']['finish-args'] = LiteralScalar('\n'.join(manifest['finish-args'])) + + return yaml.dump(yml, default_flow_style=False, indent=4) + + def _default_container_yaml(self, runtime_name, runtime_version): + pkg = self.pkg + command = pkg + branch = 'f' + str(runtime_version) + + container_yaml = dedent(f'''\ + flatpak: + # Derived from the project's domain name + id: org.example.MyApp + branch: stable + runtime-name: {runtime_name} + runtime-version: {branch} + # RPM package(s) to install, main package first + packages: + - {pkg} + # Binary to execute to run the app + command: {command} + # Not sandboxed. See 'man flatpak-build-finish' + finish-args: |- + --device=dri + --filesystem=host + --share=ipc + --socket=x11 + --socket=wayland + --socket=session-bus + ''') + + return container_yaml + + def _write_container_yaml(self, output_fname, flathub_manifest, runtime_name, runtime_version): + if flathub_manifest: + container_yaml = self._flathub_container_yaml(flathub_manifest, runtime_name, runtime_version) + else: + container_yaml = self._default_container_yaml(runtime_name, runtime_version) + + with open(output_fname, 'w') as f: + f.write(container_yaml) + + print(f"Generated container specification: {output_fname!r}." + f" Please edit appropriately.") + + def run(self, output_containerspec, + force=False, flathub=None, runtime_name=None, runtime_version=None): + flathub_manifest = _load_flathub_manifest(flathub) if flathub else None + + if output_containerspec is None: + output_containerspec = 'container.yaml' + + if not force: + if os.path.exists(output_containerspec): + raise click.ClickException(f"{output_containerspec} exists." + f" Pass --force to overwrite.") + + if runtime_name is None: + runtime_name = 'flatpak-runtime' + + if runtime_version is None: + response = requests.get("https://bodhi.fedoraproject.org/releases/?state=current") + response.raise_for_status() + + runtime_version = max(int( + r["version"]) + for r in response.json()["releases"] + if r["id_prefix"] == "FEDORA-FLATPAK" + ) + + self._write_container_yaml(output_containerspec, flathub_manifest, runtime_name, runtime_version) diff --git a/tests/files/generator/apps.json b/tests/files/generator/apps.json new file mode 100644 index 0000000..bf8df16 --- /dev/null +++ b/tests/files/generator/apps.json @@ -0,0 +1,7 @@ +[{"flatpakAppId":"org.gnome.eog", + "name":"Eye of GNOME", + "summary":"Browse and rotate images", + "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.eog.png"}, +{"flatpakAppId":"org.gnome.FeedReader", + "name":"FeedReader","summary":"RSS client for various webservices", + "iconDesktopUrl":"/repo/appstream/x86_64/icons/128x128/org.gnome.FeedReader.png"}] diff --git a/tests/files/generator/eog.json b/tests/files/generator/eog.json new file mode 100644 index 0000000..5e6824e --- /dev/null +++ b/tests/files/generator/eog.json @@ -0,0 +1,16 @@ +{ + "id": "org.gnome.eog", + "runtime": "org.gnome.Platform", + "runtime-version": "3.30", + "sdk": "org.gnome.Sdk", + "branch": "stable", + "command": "eog", + "rename-desktop-file": "eog.desktop", + "rename-appdata-file": "eog.appdata.xml", + "rename-icon": "eog", + "copy-icon": true, + "finish-args": + /* X11 + XShm access */ + ["--share=ipc", + "--socket=x11"] +} diff --git a/tests/files/generator/eog.yaml b/tests/files/generator/eog.yaml new file mode 100644 index 0000000..bac8625 --- /dev/null +++ b/tests/files/generator/eog.yaml @@ -0,0 +1,15 @@ +# eog.json has id so that we test both +app-id: org.gnome.eog +runtime: org.gnome.Platform +runtime-version: '3.30' +sdk: org.gnome.Sdk +branch: stable +command: eog +rename-desktop-file: eog.desktop +rename-appdata-file: eog.appdata.xml +rename-icon: eog +copy-icon: true +finish-args: + # X11 + XShm access + - --share=ipc + - --socket=x11 diff --git a/tests/files/generator/releases.json b/tests/files/generator/releases.json new file mode 100644 index 0000000..c016f76 --- /dev/null +++ b/tests/files/generator/releases.json @@ -0,0 +1,30 @@ +{ + "releases": [ + { + "name": "F38F", + "long_name": "Fedora 38 Flatpaks", + "version": "38", + "id_prefix": "FEDORA-FLATPAK", + "branch": "f38", + "dist_tag": "f38-flatpak", + "stable_tag": "f38-flatpak-updates", + "testing_tag": "f38-flatpak-updates-testing", + "candidate_tag": "f38-flatpak-updates-candidate", + "pending_signing_tag": "", + "pending_testing_tag": "f38-flatpak-updates-testing-pending", + "pending_stable_tag": "f38-flatpak-updates-pending", + "override_tag": "f38-flatpak-override", + "mail_template": "fedora_errata_template", + "state": "current", + "composed_by_bodhi": true, + "create_automatic_updates": false, + "package_manager": "unspecified", + "testing_repository": null, + "eol": null + } + ], + "page": 1, + "pages": 1, + "rows_per_page": 20, + "total": 1 +} diff --git a/tests/test_flatpak_generator.py b/tests/test_flatpak_generator.py new file mode 100644 index 0000000..1c4a4de --- /dev/null +++ b/tests/test_flatpak_generator.py @@ -0,0 +1,125 @@ +"""In-process tests for the flatpak generator""" + +import logging +import os +import tempfile + +from click.testing import CliRunner +import pytest +import responses +import yaml + +from flatpak_module_tools.cli import cli + + +log = logging.getLogger(__name__) + +testfiles_dir = os.path.join(os.path.dirname(__file__), 'files', 'generator') + +with open(os.path.join(testfiles_dir, 'apps.json')) as f: + APPS_JSON = f.read() + +with open(os.path.join(testfiles_dir, 'eog.yaml')) as f: + EOG_YAML = f.read() + +with open(os.path.join(testfiles_dir, 'eog.json')) as f: + EOG_JSON = f.read() + +with open(os.path.join(testfiles_dir, 'releases.json')) as f: + RELEASES_JSON = f.read() + + +def _generate_flatpak(rpm, flathub=None, runtime_name=None, runtime_version=None, expected_error_output=None): + cmd = ['init'] + cmd.append(rpm) + if flathub: + cmd += ['--flathub', flathub] + + prevdir = os.getcwd() + with tempfile.TemporaryDirectory() as workdir: + try: + os.chdir(workdir) + runner = CliRunner() + result = runner.invoke(cli, cmd, catch_exceptions=False) + if expected_error_output is not None: + assert result.exit_code != 0 + assert expected_error_output in result.output + return + else: + assert result.exit_code == 0 + + with open('container.yaml') as f: + contents = f.read() + + log.info('container.yaml:\n%s\n', contents) + container_yaml = yaml.safe_load(contents) + finally: + os.chdir(prevdir) + + return container_yaml + + +class TestFlatpak(object): + @pytest.mark.filterwarnings('ignore::DeprecationWarning:koji') + @pytest.mark.filterwarnings('ignore::PendingDeprecationWarning:koji') + @pytest.mark.needs_metadata + def test_generated_flatpak_files(self): + container_yaml = _generate_flatpak('eog') + + @responses.activate + @pytest.mark.needs_metadata + @pytest.mark.parametrize(('search_term', 'extension', 'expected_error'), + [ + ('org.gnome.eog', 'yaml', None), + ('org.gnome.eog', 'yml', None), + ('org.gnome.eog', 'json', None), + ('eYe of gNome', 'yaml', None), + ('org.gnome', 'yaml', + 'Multiple matches found on flathub.org'), + ('notexist', 'yaml', + 'No match found on flathub.org'), + ]) + def test_flatpak_from_flathub(self, search_term, extension, + expected_error): + responses.add(responses.GET, 'https://flathub.org/api/v1/apps', + body=APPS_JSON, content_type='application/json') + responses.add(responses.GET, 'https://bodhi.fedoraproject.org/releases/?state=current', + body=RELEASES_JSON, content_type='application/json') + + app_id = 'org.gnome.eog' + base = 'https://raw.githubusercontent.com/flathub' + + for ext, content_type, body in [ + ('yml', 'application/x-yaml', EOG_YAML), + ('yaml', 'application/x-yaml', EOG_YAML), + ('json', 'application/json', EOG_JSON)]: + if extension == ext: + responses.add(responses.GET, + f"{base}/{app_id}/master/{app_id}.{ext}", + body=body, content_type=content_type) + else: + responses.add(responses.GET, + f"{base}/{app_id}/master/{app_id}.{ext}", + body='Not found', status=404) + + if expected_error is None: + container_yaml = \ + _generate_flatpak('eog', + flathub=search_term, + expected_error_output=expected_error) + + f = container_yaml['flatpak'] + + assert f['id'] == 'org.gnome.eog' + assert f['command'] == 'eog' + assert f['runtime-name'] == 'flatpak-runtime' + assert f['runtime-version'] == 'f38' + assert f['rename-desktop-file'] == 'eog.desktop' + assert f['rename-appdata-file'] == 'eog.appdata.xml' + assert f['rename-icon'] == 'eog' + assert f['copy-icon'] is True + assert f['finish-args'] == '--share=ipc\n--socket=x11' + assert f['packages'][0] == 'eog' + else: + _generate_flatpak('eog', flathub=search_term, + expected_error_output=expected_error)