From 4dcabf1c62f63ca81c8f0d00f40a02650ba88952 Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Aug 29 2016 09:18:14 +0000 Subject: [PATCH 1/4] Replace dnf calls by dnf-api use The packages of modules will be handled by dnf API (mostly) --- diff --git a/fm/api_clients.py b/fm/api_clients.py index 6adb243..20a0260 100644 --- a/fm/api_clients.py +++ b/fm/api_clients.py @@ -18,15 +18,20 @@ from __future__ import print_function +import dnf +import dnf.exceptions import os +import sys import modulemd import fm.exceptions + from fm.api_client import URLAPIClient, APIClient from fm.modules import Modules from fm.repo_config_file import RepoConfigFile + class APIClients(APIClient): """ Class managing multiple APIClients instances and merging their results @@ -150,3 +155,109 @@ class APIClients(APIClient): mods.remove_old_cached_modules() return mods + + +class DnfBase: + def __init__(self): + """ + Setup DNF - read all repos, and fill - sack. + """ + self.dnfbase = dnf.Base() + self.repo_files = [] + self._dnfsetup() + + def _dnfsetup(self): + self.dnfbase.read_all_repos() + self.dnfbase.fill_sack() + + def _repo_id(self, reponame): + return '_fm_' + reponame + + def dnf_install(self, pkg_specs, module_name=None, strict=True): + """ + Mark packages given by pkg_spec from module repository for installation. + @param pkg_specs: list of pkg_specs + @param module_name - from it has to be installed: + @param strict: dnf strict options + """ + + errors = [] + for pkg_spec in pkg_specs: + try: + self.dnfbase.install( + pkg_spec, reponame=self._repo_id(module_name), strict=True) + except dnf.exceptions.MarkingError as e: + print(e) + print('No match for argument: ' + pkg_spec) + errors.append(e) + if errors and strict: + raise dnf.exceptions.MarkingError("Unable to find a match") + + def dnf_remove(self, module_name, repo_file): + """ + Mark all packages installed from module repository for installation. + @param module_name: string - name of module + @param repo_file: Object that will be deleted + """ + + done = False + self.repo_files.append(repo_file) + + # Remove all packages. + try: + self.dnfbase.remove('*', self._repo_id(module_name)) + except dnf.exceptions.MarkingError: + print('No package installed from the repository.') + else: + done = True + + if not done: + raise dnf.exceptions.Error('No packages marked for removal.') + + def dnf_upgrade(self, module_name): + """ + Upgrade all packages installed from module repository by packages from + same repository + @param module_name: string - name of module + """ + reponame = self._repo_id(module_name) + pkg_specs = [str(pkg) for pkg in self.dnfbase.sack.query().installed() + if pkg._from_repo == reponame] + + for pkg_spec in pkg_specs: + try: + self.dnfbase.upgrade(pkg_spec, reponame) + except dnf.exceptions.MarkingError: + print('No match for argument: ' + pkg_spec) + else: + done = True + if not done: + raise dnf.exceptions.Error('No packages marked for upgrade.') + + def transaction_run(self, allow_erasing=False): + """ + Perform transaction for marked packages including dep-solving + @param allow_erasing: DNF option + """ + def _remove_repofile(repo_files): + for repo_file in repo_files: + repo_file.remove() + + try: + self.dnfbase.resolve(allow_erasing=allow_erasing) + except dnf.exceptions.DepsolveError as e: + print(e) + _remove_repofile(self.repo_files) + sys.exit('Dependencies cannot be resolved.') + try: + self.dnfbase.download_packages(self.dnfbase.transaction.install_set) + except dnf.exceptions.DownloadError as e: + print(e) + _remove_repofile(self.repo_files) + sys.exit('Required package cannot be downloaded.') + # The request can finally be fulfilled. + self.dnfbase.do_transaction() + _remove_repofile(self.repo_files) + + +DNFBASE = DnfBase() diff --git a/fm/module.py b/fm/module.py index 709cce1..4009147 100644 --- a/fm/module.py +++ b/fm/module.py @@ -26,9 +26,11 @@ import stat import fm.exceptions from fm.repo_file import RepoFile +from fm.api_clients import DNFBASE from subprocess import * + class Module(object): """ Class representing Fedora Module. @@ -67,36 +69,6 @@ class Module(object): return self.api.get_metadata_expire() - def _execute_dnf(self, cmd, interactive = False): - """ - Executes the DNF command. - - :param string cmd: DNF command. - :raises fm.exceptions.Error: If DNF exits with an error code. - """ - if interactive: - ret = os.system(cmd) - if ret != 0: - raise fm.exceptions.Error( - "Cannot execute dnf command: {}".format(cmd) - ) - return - - p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True) - out, err = p.communicate() - print(out.decode("utf8")) - - if p.returncode != 0: - if out.find(b"No package installed from the repository.") != -1: - return - if out.find(b"Error: Nothing to do.") != -1: - return - - print(err.decode("utf8")) - raise fm.exceptions.Error( - "Cannot execute dnf command: {}".format(cmd) - ) - def _install_profile_rpms(self, profile_names): """ Installs RPMs defined in the Module's profile `profile_name`. @@ -116,13 +88,7 @@ class Module(object): rpms += profile.rpms - cmd = "dnf repository-packages _fm_{} install {} --allowerasing".format(self.name, ' '.join(rpms)) - if self.api.opts.assumeyes: - interactive = False - cmd += " -y" - else: - interactive = True - self._execute_dnf(cmd, True) + DNFBASE.dnf_install(rpms, module_name=self.name, strict=True) def fetch_module_metadata(self): """ @@ -154,35 +120,12 @@ class Module(object): self.repo_file.remove() raise - def get_installed_packages(self): - cmd = "dnf repository-packages _fm_{} list installed".format(self.name) - p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=True) - out, err = p.communicate() - if p.returncode != 0: - raise fm.exceptions.Error( - "Cannot get list of packages: {}".format(cmd) - ) - - pkgs = [] - data = out.decode("utf8") - for line in data.split("\n"): - if not line.endswith("_fm_{}".format(self.name)): - continue - pkg = line[:line.find(" ")] - if len(pkg) == 0: - continue - pkgs.append(pkg) - - return pkgs - def upgrade(self, call_dnf = True): """ Upgrades the packages provided by module to new release. """ if call_dnf: - installed_pkgs = self.get_installed_packages() - if len(installed_pkgs) != 0: - self._execute_dnf("dnf repository-packages _fm_{} upgrade {} -y".format(self.name, " ".join(installed_pkgs)), True) + DNFBASE.dnf_upgrade(self.name) def disable(self, call_dnf = True): """ @@ -194,8 +137,9 @@ class Module(object): :raises fm.exceptions.Error: If DNF exits with an error code. """ if call_dnf: - self._execute_dnf("dnf repository-packages _fm_{} remove -y".format(self.name)) - self.repo_file.remove() + DNFBASE.dnf_remove(self.name, self.repo_file) + else: + self.repo_file.remove() def is_enabled(self): """ From b594f0815ba61a5554e7e6f104503c50b39dc02f Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Aug 29 2016 09:18:14 +0000 Subject: [PATCH 2/4] Add dnf transaction overview It uses non-api of DNF and it is applied if fm is run as standalone. --- diff --git a/fm/api_clients.py b/fm/api_clients.py index 20a0260..68ca47b 100644 --- a/fm/api_clients.py +++ b/fm/api_clients.py @@ -19,6 +19,7 @@ from __future__ import print_function import dnf +import dnf.cli.output import dnf.exceptions import os import sys @@ -163,6 +164,7 @@ class DnfBase: Setup DNF - read all repos, and fill - sack. """ self.dnfbase = dnf.Base() + self.output = dnf.cli.output.Output(self.dnfbase, self.dnfbase.conf) self.repo_files = [] self._dnfsetup() @@ -255,6 +257,7 @@ class DnfBase: print(e) _remove_repofile(self.repo_files) sys.exit('Required package cannot be downloaded.') + print(self.output.list_transaction(self.dnfbase.transaction)) # The request can finally be fulfilled. self.dnfbase.do_transaction() _remove_repofile(self.repo_files) From ae1f4287c23b0adbbfed751db0cd2371ba40adbb Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Aug 29 2016 09:18:14 +0000 Subject: [PATCH 3/4] fixup! Replace dnf calls by dnf-api use --- diff --git a/fm/api_clients.py b/fm/api_clients.py index 68ca47b..a95b3bd 100644 --- a/fm/api_clients.py +++ b/fm/api_clients.py @@ -163,26 +163,33 @@ class DnfBase: """ Setup DNF - read all repos, and fill - sack. """ + # Todo - dnf.Base() should be replaced by dnf.cli.base available for + # plugins commands. self.dnfbase = dnf.Base() + # Todo - remove when dnf.cli.base is used - this is no API self.output = dnf.cli.output.Output(self.dnfbase, self.dnfbase.conf) self.repo_files = [] - self._dnfsetup() + self.allow_erasing = False def _dnfsetup(self): - self.dnfbase.read_all_repos() - self.dnfbase.fill_sack() + if not self.dnfbase.sack: + self.dnfbase.read_all_repos() + self.dnfbase.fill_sack() def _repo_id(self, reponame): return '_fm_' + reponame - def dnf_install(self, pkg_specs, module_name=None, strict=True): + def dnf_install(self, pkg_specs, module_name=None, strict=True, + allow_erasing = False): """ Mark packages given by pkg_spec from module repository for installation. @param pkg_specs: list of pkg_specs @param module_name - from it has to be installed: @param strict: dnf strict options """ - + if allow_erasing: + self.allow_erasing = allow_erasing + self._dnfsetup() errors = [] for pkg_spec in pkg_specs: try: @@ -195,13 +202,16 @@ class DnfBase: if errors and strict: raise dnf.exceptions.MarkingError("Unable to find a match") - def dnf_remove(self, module_name, repo_file): + def dnf_remove(self, module_name, repo_file, allow_erasing = False): """ Mark all packages installed from module repository for installation. @param module_name: string - name of module @param repo_file: Object that will be deleted """ + if allow_erasing: + self.allow_erasing = allow_erasing + self._dnfsetup() done = False self.repo_files.append(repo_file) @@ -216,15 +226,19 @@ class DnfBase: if not done: raise dnf.exceptions.Error('No packages marked for removal.') - def dnf_upgrade(self, module_name): + def dnf_upgrade(self, module_name, allow_erasing = False): """ Upgrade all packages installed from module repository by packages from same repository @param module_name: string - name of module """ + if allow_erasing: + self.allow_erasing = allow_erasing + + self._dnfsetup() reponame = self._repo_id(module_name) pkg_specs = [str(pkg) for pkg in self.dnfbase.sack.query().installed() - if pkg._from_repo == reponame] + if pkg.from_repo == reponame] for pkg_spec in pkg_specs: try: @@ -236,7 +250,7 @@ class DnfBase: if not done: raise dnf.exceptions.Error('No packages marked for upgrade.') - def transaction_run(self, allow_erasing=False): + def transaction_run(self): """ Perform transaction for marked packages including dep-solving @param allow_erasing: DNF option @@ -246,18 +260,18 @@ class DnfBase: repo_file.remove() try: - self.dnfbase.resolve(allow_erasing=allow_erasing) + self.dnfbase.resolve(allow_erasing=self.allow_erasing) except dnf.exceptions.DepsolveError as e: print(e) _remove_repofile(self.repo_files) sys.exit('Dependencies cannot be resolved.') + print(self.output.list_transaction(self.dnfbase.transaction)) try: self.dnfbase.download_packages(self.dnfbase.transaction.install_set) except dnf.exceptions.DownloadError as e: print(e) _remove_repofile(self.repo_files) sys.exit('Required package cannot be downloaded.') - print(self.output.list_transaction(self.dnfbase.transaction)) # The request can finally be fulfilled. self.dnfbase.do_transaction() _remove_repofile(self.repo_files) diff --git a/fm/fm_modules_resolver.py b/fm/fm_modules_resolver.py index fa98fe0..486a2c1 100644 --- a/fm/fm_modules_resolver.py +++ b/fm/fm_modules_resolver.py @@ -169,6 +169,8 @@ class FmModulesResolver(ModulesResolver): self._disable_modules(ret.to_disable) self._upgrade_modules(ret.to_upgrade, profiles = profiles) self._upgrade_modules(ret.to_downgrade, profiles = profiles) + if fm.api_clients.DNFBASE.dnfbase.sack: + fm.api_clients.DNFBASE.transaction_run() def execute(self, action, arg, profiles = ["default"]): """ diff --git a/fm/module.py b/fm/module.py index 4009147..8656c31 100644 --- a/fm/module.py +++ b/fm/module.py @@ -24,9 +24,10 @@ import re import os import stat import fm.exceptions +import dnf from fm.repo_file import RepoFile -from fm.api_clients import DNFBASE +import fm.api_clients from subprocess import * @@ -88,7 +89,9 @@ class Module(object): rpms += profile.rpms - DNFBASE.dnf_install(rpms, module_name=self.name, strict=True) + fm.api_clients.DNFBASE.dnf_install(rpms, module_name=self.name, + strict=True, + allow_erasing = True) def fetch_module_metadata(self): """ @@ -125,7 +128,7 @@ class Module(object): Upgrades the packages provided by module to new release. """ if call_dnf: - DNFBASE.dnf_upgrade(self.name) + fm.api_clients.DNFBASE.dnf_upgrade(self.name) def disable(self, call_dnf = True): """ @@ -137,7 +140,11 @@ class Module(object): :raises fm.exceptions.Error: If DNF exits with an error code. """ if call_dnf: - DNFBASE.dnf_remove(self.name, self.repo_file) + try: + fm.api_clients.DNFBASE.dnf_remove(self.name, self.repo_file) + except dnf.exceptions.Error as err: + if str(err) != "No packages marked for removal.": + raise else: self.repo_file.remove() From 3101cc9ffbc1c11e5f016f9289ba0fcb1f6952ff Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Aug 29 2016 09:18:14 +0000 Subject: [PATCH 4/4] Use plugin.base if it is available It use full power of dnf plugins if module is run as dnf command. There is still issue with module remove where module is remove before transaction is performed. --- diff --git a/fm/api_clients.py b/fm/api_clients.py index a95b3bd..0648d3e 100644 --- a/fm/api_clients.py +++ b/fm/api_clients.py @@ -168,18 +168,32 @@ class DnfBase: self.dnfbase = dnf.Base() # Todo - remove when dnf.cli.base is used - this is no API self.output = dnf.cli.output.Output(self.dnfbase, self.dnfbase.conf) - self.repo_files = [] + self.repo_files = {'enabling': [], 'disabling': []} + # if self.pluginbase - allow_erasing setting is used from dnf.conf self.allow_erasing = False + self.pluginbase = False def _dnfsetup(self): if not self.dnfbase.sack: + if self.pluginbase: + self.dnfbase.repos.clear() self.dnfbase.read_all_repos() self.dnfbase.fill_sack() def _repo_id(self, reponame): return '_fm_' + reponame - def dnf_install(self, pkg_specs, module_name=None, strict=True, + def repofiles_action(self, repo_files_action): + if repo_files_action == 'roll_back': + for repo_file in self.repo_files['enabling']: + repo_file.remove() + for repo_file in self.repo_files['disabling']: + repo_file.create() + if repo_files_action == 'disabling': + for repo_file in self.repo_files['disabling']: + repo_file.remove() + + def dnf_install(self, pkg_specs, module_name, repo_file, strict=True, allow_erasing = False): """ Mark packages given by pkg_spec from module repository for installation. @@ -187,6 +201,7 @@ class DnfBase: @param module_name - from it has to be installed: @param strict: dnf strict options """ + self.repo_files['enabling'].append(repo_file) if allow_erasing: self.allow_erasing = allow_erasing self._dnfsetup() @@ -208,12 +223,12 @@ class DnfBase: @param module_name: string - name of module @param repo_file: Object that will be deleted """ + self.repo_files['disabling'].append(repo_file) if allow_erasing: self.allow_erasing = allow_erasing self._dnfsetup() done = False - self.repo_files.append(repo_file) # Remove all packages. try: @@ -255,26 +270,23 @@ class DnfBase: Perform transaction for marked packages including dep-solving @param allow_erasing: DNF option """ - def _remove_repofile(repo_files): - for repo_file in repo_files: - repo_file.remove() try: self.dnfbase.resolve(allow_erasing=self.allow_erasing) except dnf.exceptions.DepsolveError as e: print(e) - _remove_repofile(self.repo_files) + self.repofiles_action('roll_back') sys.exit('Dependencies cannot be resolved.') print(self.output.list_transaction(self.dnfbase.transaction)) try: self.dnfbase.download_packages(self.dnfbase.transaction.install_set) except dnf.exceptions.DownloadError as e: print(e) - _remove_repofile(self.repo_files) + self.repofiles_action('roll_back') sys.exit('Required package cannot be downloaded.') # The request can finally be fulfilled. self.dnfbase.do_transaction() - _remove_repofile(self.repo_files) + self.repofiles_action('disabling') DNFBASE = DnfBase() diff --git a/fm/fm_modules_resolver.py b/fm/fm_modules_resolver.py index 486a2c1..a94d283 100644 --- a/fm/fm_modules_resolver.py +++ b/fm/fm_modules_resolver.py @@ -169,7 +169,8 @@ class FmModulesResolver(ModulesResolver): self._disable_modules(ret.to_disable) self._upgrade_modules(ret.to_upgrade, profiles = profiles) self._upgrade_modules(ret.to_downgrade, profiles = profiles) - if fm.api_clients.DNFBASE.dnfbase.sack: + if fm.api_clients.DNFBASE.dnfbase.sack \ + and not fm.api_clients.DNFBASE.pluginbase: fm.api_clients.DNFBASE.transaction_run() def execute(self, action, arg, profiles = ["default"]): diff --git a/fm/module.py b/fm/module.py index 8656c31..76315d5 100644 --- a/fm/module.py +++ b/fm/module.py @@ -89,7 +89,8 @@ class Module(object): rpms += profile.rpms - fm.api_clients.DNFBASE.dnf_install(rpms, module_name=self.name, + fm.api_clients.DNFBASE.dnf_install(rpms, self.name, + self.repo_file, strict=True, allow_erasing = True) diff --git a/plugins/module.py b/plugins/module.py index 91cc2f1..f51d5c9 100644 --- a/plugins/module.py +++ b/plugins/module.py @@ -63,6 +63,7 @@ import stat from fm.cli import Cli from fm.modules import Modules import fm.exceptions +import fm.api_clients YES = set([_('yes'), _('y')]) NO = set([_('no'), _('n'), '']) @@ -137,12 +138,54 @@ class ModuleCommand(dnf.cli.Command): module search httpd """) + def configure(self, args): + # setup resolving + demands = self.cli.demands + demands.resolving = False + + def run_transaction(self): + """Perform the depsolve, download and RPM transaction stage.""" + # Solve problem with incorrect module repo file committing + if self.base.sack: + if self.base.transaction is None: + self.base.resolve(self.cli.demands.allow_erasing) + logger.info(_('Dependencies resolved.')) + + self.base._plugins.run_resolved() + + # Run the transaction + displays = [] + if self.cli.demands.transaction_display is not None: + displays.append(self.cli.demands.transaction_display) + try: + self.base.do_transaction(display=displays) + except dnf.cli.CliError as exc: + logger.error(ucd(exc)) + fm.api_clients.DNFBASE.repofiles_action('roll_back') + return 1 + except dnf.exceptions.TransactionCheckError as err: + fm.api_clients.DNFBASE.repofiles_action('roll_back') + for msg in self.cli.command.get_error_output(err): + logger.critical(msg) + except IOError as e: + fm.api_clients.DNFBASE.repofiles_action('roll_back') + raise e + else: + self.base._plugins.run_transaction() + logger.info(_('Complete!')) + fm.api_clients.DNFBASE.repofiles_action('disabling') + return 0 + + def run(self, extcmds): """ Executes subcommand passed to 'dnf' command. :param list extcmds: list List of subcommands. """ + fm.api_clients.DNFBASE.dnfbase = self.base + fm.api_clients.DNFBASE.pluginbase = True + try: cli = Cli() cli.run(extcmds)