From 0ae379874e53923e75d9d3b849179854391e2255 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 04 2017 10:17:55 +0000 Subject: [PATCH 1/7] fixed issues with deleting --- diff --git a/moduleframework/common.py b/moduleframework/common.py index e2d004c..f328dbf 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -105,6 +105,7 @@ trans_dict = {"HOSTIPADDR": hostipaddr, } +BASEPATHDIR = "/opt" PDCURL = "https://pdc.fedoraproject.org/rest_api/v1/unreleasedvariants" URLBASECOMPOSE = "https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server" REPOMD = "repodata/repomd.xml" diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index de3264b..4adb371 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -731,6 +731,7 @@ class NspawnHelper(RpmHelper): relative change root path """ super(NspawnHelper, self).__init__() + self.baseprefix = os.path.join(BASEPATHDIR, "chroot_") self.__selinuxState = None time.time() actualtime = time.time() @@ -738,9 +739,7 @@ class NspawnHelper(RpmHelper): self.jmeno = "%s_%r" % (self.moduleName, actualtime) else: self.jmeno = self.moduleName - self.chrootpath = os.path.abspath( - os.path.join( - "/opt", "chroot_%s" % self.jmeno)) + self.chrootpath = os.path.abspath(self.baseprefix + self.jmeno) print_info("name of CHROOT directory:", self.chrootpath) trans_dict["ROOT"] = self.chrootpath @@ -786,20 +785,37 @@ class NspawnHelper(RpmHelper): return True raise NspawnExc("Unable to start machine %s within %d" % (self.jmeno, DEFAULTRETRYTIMEOUT)) - def __prepareSetup(self): + def __do_smart_start_cleanup(self): """ Internal method, do not use it anyhow :return: None """ - if get_if_do_cleanup() and os.path.exists(self.chrootpath): - shutil.rmtree(self.chrootpath, ignore_errors=True) - os.mkdir(self.chrootpath) + + if get_if_do_cleanup(): + # delete directory with same same (in case used option DO NOT CLEANUP) + if os.path.exists(self.chrootpath): + shutil.rmtree(self.chrootpath, ignore_errors=True) + # DELETE every chroot dir in case any exists + dirstodelete = glob.glob(self.baseprefix + "*") + if dirstodelete: + for dtd in dirstodelete: + shutil.rmtree(dtd, ignore_errors=True) + # Terminate machine in case of same name and still running try: self.runHost("machinectl terminate %s" % self.jmeno, verbose=is_debug(), ignore_status=True) self.__is_killed() except BaseException: pass + os.mkdir(self.chrootpath) + + def __prepareSetup(self): + """ + Internal method, do not use it anyhow + + :return: None + """ + self.__do_smart_start_cleanup() if not os.path.exists(os.path.join(self.chrootpath, "usr")): self.runHost("{HOSTPACKAGER} install systemd-container", verbose=is_not_silent()) repos_to_use = "" @@ -1023,7 +1039,14 @@ gpgcheck=0 self.__is_killed() except Exception as poweroffex: print_info("Unable to stop machine via poweroff, terminating", poweroffex) - self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True) + try: + time.sleep(1) + self.runHost("machinectl terminate %s" % self.jmeno, ignore_status=True) + self.__is_killed() + except Exception as poweroffexterm: + print_info("Unable to stop machine via terminate, STRANGE", poweroffexterm) + time.sleep(DEFAULTRETRYTIMEOUT) + pass pass if not os.environ.get('MTF_SKIP_DISABLING_SELINUX'): diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index d43f61b..c3e68ac 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -245,7 +245,7 @@ class PDCParser(): :return: str """ - dir_prefix = "/opt" + dir_prefix = BASEPATHDIR utils.process.run("{HOSTPACKAGER} install createrepo koji".format( **trans_dict), ignore_status=True) if is_recursive_download(): From fbfffd3419a3331217bdf8fa412fa56ddfac67d1 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 04 2017 10:17:55 +0000 Subject: [PATCH 2/7] koji package downloading cleanup, moved to pdc instead of hardcoding in MTF main file --- diff --git a/moduleframework/common.py b/moduleframework/common.py index f328dbf..de22390 100644 --- a/moduleframework/common.py +++ b/moduleframework/common.py @@ -167,3 +167,36 @@ def is_recursive_download(): :return: bool """ return bool(os.environ.get("MTF_RECURSIVE_DOWNLOAD")) + + +def get_if_do_cleanup(): + """ + Returns boolean value in case variable is set. + It is used internally in code + + :return: bool + """ + cleanup = os.environ.get('MTF_DO_NOT_CLEANUP') + return not bool(cleanup) + + +def get_if_remoterepos(): + """ + Returns boolean value in case variable is set. + It is used internally in code + + :return: bool + """ + rreps = os.environ.get('MTF_REMOTE_REPOS') + return bool(rreps) + + +def get_if_module(): + """ + Returns boolean value in case variable is set. + It is used internally in code + + :return: bool + """ + rreps = os.environ.get('MTF_DISABLE_MODULE') + return not bool(rreps) diff --git a/moduleframework/module_framework.py b/moduleframework/module_framework.py index 4adb371..f46eec3 100755 --- a/moduleframework/module_framework.py +++ b/moduleframework/module_framework.py @@ -1464,42 +1464,6 @@ def get_latest_repo_url(wmodule="base-runtime", wstream="master", fake=False): if fake: return "http://mirror.vutbr.cz/fedora/releases/25/Everything/x86_64/os/" else: - localrepo = pdc_data.PDCParser() - localrepo.setLatestPDC(wmodule, wstream) - if get_if_remoterepos(): - return localrepo.generateRepoUrl() - else: - return localrepo.createLocalRepoFromKoji() - - -def get_if_do_cleanup(): - """ - Returns boolean value in case variable is set. - It is used internally in code - - :return: bool - """ - cleanup = os.environ.get('MTF_DO_NOT_CLEANUP') - return not bool(cleanup) - - -def get_if_remoterepos(): - """ - Returns boolean value in case variable is set. - It is used internally in code - - :return: bool - """ - rreps = os.environ.get('MTF_REMOTE_REPOS') - return bool(rreps) - - -def get_if_module(): - """ - Returns boolean value in case variable is set. - It is used internally in code - - :return: bool - """ - rreps = os.environ.get('MTF_DISABLE_MODULE') - return not bool(rreps) + tmp_pdc = pdc_data.PDCParser() + tmp_pdc.setLatestPDC(wmodule, wstream) + return tmp_pdc.generateRepoUrl() diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index c3e68ac..b9105f3 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -152,8 +152,11 @@ class PDCParser(): """ # rpmrepo = "http://kojipkgs.fedoraproject.org/repos/%s/latest/%s" % ( # self.pdcdata["koji_tag"] + "-build", ARCH) - rpmrepo = "%s/%s/os/" % (URLBASECOMPOSE, ARCH) - return rpmrepo + if get_if_remoterepos(): + rpmrepo = "%s/%s/os/" % (URLBASECOMPOSE, ARCH) + return rpmrepo + else: + return self.createLocalRepoFromKoji() def generateGitHash(self): """ @@ -271,15 +274,3 @@ class PDCParser(): (absdir, absdir), shell=True, verbose=is_debug()) return "file://%s" % absdir - def generateParamsLocalKojiPkgs(self): - """ - Return list of params what has to be set for automation like (local repo): - MODULE=nspawn MODULEMDURL=file:///... URL=file:///localrepo - - :return: list - """ - output = [] - output.append("URL=%s" % self.createLocalRepoFromKoji()) - output.append("MODULEMDURL=%s" % self.generateModuleMDFile()) - output.append("MODULE=%s" % "nspawn") - return output diff --git a/tools/run-them.sh b/tools/run-them.sh index 9193d2c..94253e0 100755 --- a/tools/run-them.sh +++ b/tools/run-them.sh @@ -21,16 +21,6 @@ # Authors: Jan Scotka # -SLEEP=15 - -function sleep_a_while(){ - echo "sleep for $SLEEP minutes to ensure that repos are ready" - for foo in `seq $SLEEP`; do - sleep 60 - echo "$foo/$SLEEP minutes done" - done -} - function fedpkg_alt(){ #LFILE="alt_fedpkg.conf" #cat /etc/rpkg/fedpkg.conf |sed 's/anongiturl.*/anongiturl = https:\/\/src.fedoraproject.org\/git\/%(module)s/' > $LFILE @@ -54,12 +44,13 @@ export AVOCADOCMD="avocado run --xunit $XUFILE --show-job-log" export RESULTTOOLS=0 export MTF_RECURSIVE_DOWNLOAD=yes + function getparams_int(){ ADDIT="$1" if [ "$PARSEITEMTYPE" = "" -o "$PARSEITEMTYPE" = "fedmsg" ]; then - python $MTF_PATH/tools/taskotron-msg-reader.py -f $PARSEITEM --localrepo $ADDIT + python $MTF_PATH/tools/taskotron-msg-reader.py -f $PARSEITEM $ADDIT elif [ "$PARSEITEMTYPE" = "taskotron" -o "$PARSEITEMTYPE" = "pdc" ]; then - python $MTF_PATH/tools/taskotron-msg-reader.py -r $PARSEITEM --localrepo $ADDIT + python $MTF_PATH/tools/taskotron-msg-reader.py -r $PARSEITEM $ADDIT elif [ -z $ADDIT -a "$PARSEITEMTYPE" = "compose" ]; then python $MTF_PATH/tools/compose_info_parser.py -c $PARSEITEM -m $MODULENAME fi diff --git a/tools/taskotron-msg-reader.py b/tools/taskotron-msg-reader.py index ce05dd6..77ec63e 100755 --- a/tools/taskotron-msg-reader.py +++ b/tools/taskotron-msg-reader.py @@ -41,12 +41,6 @@ if __name__ == '__main__': parser.add_option("-l", "--latest", dest="latest", help="Use latest bits, build by MBS and stored in PDC") parser.add_option( - "--localrepo", - dest="localrepo", - action="store_true", - default=False, - help="generate local repository based on koji tag from PDC") - parser.add_option( "--commit", dest="commit", action="store_true", @@ -55,10 +49,6 @@ if __name__ == '__main__': a = PDCParser() (options, args) = parser.parse_args() - if options.localrepo: - callfnc = "generateParamsLocalKojiPkgs" - else: - callfnc = "generateParams" if options.filename: flh = open(options.filename) stdinput = "".join(flh.readlines()).strip() @@ -74,4 +64,4 @@ if __name__ == '__main__': if options.commit: print a.generateGitHash() else: - print " ".join(getattr(a, callfnc)()) + print " ".join(a.generateParams()) From db091e3c83b465dff9fea711d455ed5caa14c0fc Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 04 2017 11:03:38 +0000 Subject: [PATCH 3/7] changed testing module to minimal config. mksh is not in compose --- diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile index 7a2aca2..2ca8869 100644 --- a/examples/testing-module/Makefile +++ b/examples/testing-module/Makefile @@ -5,11 +5,11 @@ export DEBUG=yes export DOCKERFILE=./Dockerfile check-docker: - MODULE=docker $(CMD) $(TESTS) + MODULE=docker CONFIG=./fullconfig.yaml $(CMD) $(TESTS) @true check-minimal-config-docker: - MODULE=docker CONFIG=./minimal.yaml $(CMD) $(TESTS) + MODULE=docker $(CMD) $(TESTS) @true diff --git a/examples/testing-module/config.yaml b/examples/testing-module/config.yaml index c74964b..71ca43d 120000 --- a/examples/testing-module/config.yaml +++ b/examples/testing-module/config.yaml @@ -1 +1 @@ -../../docs/example-config.yaml \ No newline at end of file +../../docs/example-config-minimal.yaml \ No newline at end of file diff --git a/examples/testing-module/fullconfig.yaml b/examples/testing-module/fullconfig.yaml new file mode 120000 index 0000000..c74964b --- /dev/null +++ b/examples/testing-module/fullconfig.yaml @@ -0,0 +1 @@ +../../docs/example-config.yaml \ No newline at end of file diff --git a/examples/testing-module/minimal.yaml b/examples/testing-module/minimal.yaml deleted file mode 120000 index 71ca43d..0000000 --- a/examples/testing-module/minimal.yaml +++ /dev/null @@ -1 +0,0 @@ -../../docs/example-config-minimal.yaml \ No newline at end of file From 1616ff34db4a1b3687220193b8ff13a352241755 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 04 2017 11:41:27 +0000 Subject: [PATCH 5/7] minimal config path fix --- diff --git a/examples/testing-module/Makefile b/examples/testing-module/Makefile index 2ca8869..2a4217c 100644 --- a/examples/testing-module/Makefile +++ b/examples/testing-module/Makefile @@ -14,11 +14,11 @@ check-minimal-config-docker: check-rpm: - MODULE=nspawn $(CMD) $(TESTS) + MODULE=nspawn CONFIG=./fullconfig.yaml $(CMD) $(TESTS) @true check-minimal-config-rpm: - MODULE=nspawn CONFIG=./minimal.yaml $(CMD) $(TESTS) + MODULE=nspawn $(CMD) $(TESTS) @true check-behave-docker: diff --git a/tools/run-them.sh b/tools/run-them.sh index 94253e0..ce8f6fe 100755 --- a/tools/run-them.sh +++ b/tools/run-them.sh @@ -33,8 +33,9 @@ export PARSEITEM=$2 # compose fedmsg or None same as fedmsg export PARSEITEMTYPE=$3 export SELECTION=$4 -export MTF_PATH=/usr/share/moduleframework +export MTF_PATH="/usr/share/moduleframework" export MODULE_LINT="$MTF_PATH/tools/modulelint/*.py" +export MINIMAL_CONFIG="$MTF_PATH/docs/example-config-minimal.yaml" export MODULE_TESTS="*.py *.sh" export AVDIR=~/avocado @@ -44,7 +45,6 @@ export AVOCADOCMD="avocado run --xunit $XUFILE --show-job-log" export RESULTTOOLS=0 export MTF_RECURSIVE_DOWNLOAD=yes - function getparams_int(){ ADDIT="$1" if [ "$PARSEITEMTYPE" = "" -o "$PARSEITEMTYPE" = "fedmsg" ]; then @@ -110,7 +110,7 @@ function run_modulelint(){ TESTS="`ls $MODULE_LINT`" echo "RUN AT LEAST MODULE LINTER: $TESTS" - eval $PARAMS CONFIG=$MTF_PATH/docs/example-config-minimal.yaml $AVOCADOCMD $TESTS + eval $PARAMS CONFIG=$MINIMAL_CONFIG $AVOCADOCMD $TESTS } set -x From bc67e1c7b5bad851218d8860c1e9fff22c55b9c5 Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 04 2017 12:11:11 +0000 Subject: [PATCH 6/7] make docker linter faster, not need to invoke parent setup in own setup class, because it does offline checking --- diff --git a/tools/modulelint.py b/tools/modulelint.py index 245a452..122f379 100644 --- a/tools/modulelint.py +++ b/tools/modulelint.py @@ -39,7 +39,6 @@ class DockerfileLinter(module_framework.AvocadoTest): def setUp(self): # it is not intended just for docker, but just docker packages are # actually properly signed - super(self.__class__, self).setUp() self.dp = dockerlinter.DockerfileLinter(os.path.join(os.getcwd(), "..")) if self.dp.dockerfile is None: self.skip() From 2d60093fe2e8a1a012a86e57328853dddd26999c Mon Sep 17 00:00:00 2001 From: Jan Scotka Date: Jul 07 2017 08:43:15 +0000 Subject: [PATCH 7/7] Added unitetest to pdc module --- diff --git a/moduleframework/pdc_data.py b/moduleframework/pdc_data.py index b9105f3..354070e 100644 --- a/moduleframework/pdc_data.py +++ b/moduleframework/pdc_data.py @@ -217,7 +217,7 @@ class PDCParser(): :param dirname: string :return: None """ - print_info("DOWLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version)) + print_info("DOWNLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version)) for foo in utils.process.run("koji list-tagged --quiet %s" % self.pdcdata["koji_tag"], verbose=is_debug()).stdout.split("\n"): pkgbouid = foo.strip().split(" ")[0] if len(pkgbouid) > 4: @@ -239,7 +239,7 @@ class PDCParser(): 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() - print_info("DOWLOADING finished") + print_info("DOWNLOADING finished") def createLocalRepoFromKoji(self): """ @@ -274,3 +274,14 @@ class PDCParser(): (absdir, absdir), shell=True, verbose=is_debug()) return "file://%s" % absdir +if __name__ == "__main__": + a = PDCParser() + a.setLatestPDC(name="memcached", stream="f26") + dependencies = a.generateDepModules() + get_if_remoterepos = (lambda: True) + assert dependencies == {'base-runtime': 'f26', 'shared-userspace': 'f26', 'perl': 'f26'} + assert "https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server/x86_64/os/" == a.generateRepoUrl() + assert "URL=https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server/x86_64/os/" in a.generateParams() + assert "MODULE=nspawn" in a.generateParams() + assert len(a.generateGitHash()) == 41 + assert "Memcached is a high-performance, distributed" in a.getmoduleMD()['data']['description']