From 4320ec476f62af6c6fc3709db58f0038be447e94 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Nov 20 2017 16:47:55 +0000
Subject: [PATCH 1/2] Add coverage capturing to test suite


Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

diff --git a/ipsilon/install/ipsilon-server-install b/ipsilon/install/ipsilon-server-install
index 1a4efcb..0e1c8f9 100755
--- a/ipsilon/install/ipsilon-server-install
+++ b/ipsilon/install/ipsilon-server-install
@@ -95,6 +95,11 @@ def install(plugins, args):
         shutil.move(idp_conf, '%s.backup.%s' % (idp_conf, now))
     if not os.path.exists(instance_conf):
         os.makedirs(instance_conf, 0700)
+    coverage = ''
+    if args['coverage'] or args['branch_coverage']:
+        coverage += 'coverage.enabled = True'
+        if args['coverage_branch']:
+            coverage += '\ncoverage.branch = True'
     confopts = {'instance': args['instance'],
                 'instanceurl': args['instanceurl'],
                 'needs_mount': args.get('needs_mount'),
@@ -118,7 +123,8 @@ def install(plugins, args):
                     'database_url'] % {'datadir': args['data_dir'],
                                        'dbname': 'saml2sessions'},
                 'secure': "False" if args['secure'] == "no" else "True",
-                'debugging': "True" if args['server_debugging'] else "False"}
+                'debugging': "True" if args['server_debugging'] else "False",
+                'coverage': coverage}
     # Testing database sessions
     if 'session_type' in args:
         confopts['sesstype'] = args['session_type']
@@ -424,6 +430,10 @@ def parse_args(plugins):
     parser.add_argument('--session-timeout', default=30, type=int,
                         help='Time that sessions are valid for (in minutes, ' +
                              'default: 30 minutes)')
+    parser.add_argument('--coverage', default=False, action='store_true',
+                        help=argparse.SUPPRESS)
+    parser.add_argument('--coverage-branch', default=False,
+                        action='store_true', help=argparse.SUPPRESS)
 
     lms = []
     azs = []
diff --git a/ipsilon/ipsilon b/ipsilon/ipsilon
index 1e775a2..8fa2bf7 100755
--- a/ipsilon/ipsilon
+++ b/ipsilon/ipsilon
@@ -15,6 +15,12 @@ import os
 import atexit
 import string
 import cherrypy
+
+try:
+    import coverage
+except ImportError:  # pragma: no cover
+    coverage = None
+
 from ipsilon import find_config
 from ipsilon.util.data import AdminStore
 from ipsilon.util import page
@@ -41,6 +47,19 @@ cherrypy.lib.sessions.SqlSession = ipsilon.util.sessions.SqlSession
 cherrypy.lib.sessions.EtcdSession = ipsilon.util.sessions.EtcdSession
 cherrypy.config.update(cfgfile)
 
+# If requested, let's capture coverage data
+if cherrypy.config.get('coverage.enabled', False):
+    if not coverage:  # pragma: no cover
+        raise Exception('Coverage.py was unavailable')
+
+    cov = coverage.Coverage(
+        data_file='coverage',
+        data_suffix=True,
+        auto_data=True,
+        branch=cherrypy.config.get('coverage.branch', False))
+    cov.start()
+    atexit.register(cov.stop)
+
 # Force cherrypy logging to work. Note that this ignores the config-file
 # setting.
 cherrypy.log.screen = True
diff --git a/templates/install/ipsilon.conf b/templates/install/ipsilon.conf
index 6388619..211403f 100644
--- a/templates/install/ipsilon.conf
+++ b/templates/install/ipsilon.conf
@@ -21,3 +21,5 @@ tools.sessions.path = "${instanceurl}"
 tools.sessions.timeout = ${session_timeout}
 tools.sessions.httponly = ${secure}
 tools.sessions.secure = ${secure}
+
+${coverage}
diff --git a/tests/helpers/common.py b/tests/helpers/common.py
index e366db2..20308e3 100755
--- a/tests/helpers/common.py
+++ b/tests/helpers/common.py
@@ -9,6 +9,7 @@ import pwd
 import shutil
 import signal
 import random
+import time
 from string import Template
 import subprocess
 
@@ -87,6 +88,8 @@ class IpsilonTestBase(object):
         self.print_cases = False
         self.stdout = None
         self.stderr = None
+        self.coverage = False
+        self.coverage_branch = False
 
     def platform_supported(self):
         """This return whether the current platform supports this test.
@@ -164,6 +167,10 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir,
         newconf.add_section('arguments')
         for k in args_opts:
             newconf.set('arguments', k, args_opts[k])
+        if self.coverage:
+            newconf.set('arguments', 'coverage', True)
+        if self.coverage_branch:
+            newconf.set('arguments', 'coverage_branch', True)
 
         profile = io.BytesIO()
         newconf.write(profile)
@@ -246,7 +253,13 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir,
         cmd = [os.path.join(self.rootdir,
                             'ipsilon/install/ipsilon-server-install'),
                '--config-profile=%s' % profile]
-        subprocess.check_call(cmd, env=env,
+        if self.coverage:
+            env['COVERAGE_FILE'] = 'installer.coverage'
+            covcmd = ['coverage', 'run']
+            if self.coverage_branch:
+                covcmd.append('--branch')
+            cmd = covcmd + cmd
+        subprocess.check_call(cmd, env=env, cwd=self.testdir,
                               stdout=self.stdout, stderr=self.stderr)
         os.symlink(os.path.join(self.rootdir, 'ipsilon'),
                    os.path.join(self.testdir, 'lib', name, 'ipsilon'))
@@ -435,9 +448,22 @@ basicConstraints = CA:false""" % {'certdir': os.path.join(self.testdir,
             if ksetup.returncode != 0:
                 raise ValueError('kinit %s failed' % self.testuser)
 
+    def wait_process(self, p):
+        start = int(time.time())
+        while True:
+            if int(time.time()) - start > 5:
+                print('Process %s did not terminate in time, killing' % p.pid)
+                p.kill()
+                return
+            if p.poll() is not None:
+                return
+            time.sleep(0.1)
+
     def wait(self):
         for p in self.processes:
             os.killpg(p.pid, signal.SIGTERM)
+        for p in self.processes:
+            self.wait_process(p)
 
     def setup_servers(self, env=None):
         raise NotImplementedError()
diff --git a/tests/tests.py b/tests/tests.py
index e5a688f..5986d23 100755
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -9,6 +9,7 @@ import pkg_resources  # pylint: disable=unused-import
 
 import argparse
 from ipsilon.util import plugin
+import glob
 import os
 import sys
 import subprocess
@@ -43,6 +44,13 @@ def parse_args():
                         help='Test results header')
     parser.add_argument('--path', default='%s/testdir' % os.getcwd(),
                         help="Directory in which tests are run")
+    parser.add_argument('--coverage', default=False, action='store_true',
+                        help="Whether or not to collect coverage reports")
+    parser.add_argument('--coverage-branch', default=False,
+                        action='store_true',
+                        help='Whether to do branch coverage')
+    parser.add_argument('--coverage-min', default=0,
+                        help='Minimum coverage % to consider success')
     parser.add_argument('--fail-on-first-error', '-x', action='store_true',
                         help='Abort test run on first test failure')
     parser.add_argument('--test', action='append', default=None,
@@ -105,6 +113,8 @@ def run_test(testname, test, args):
         devnull = open(os.devnull, 'w')
         test.stdout = devnull
         test.stderr = devnull
+    test.coverage = args['coverage']
+    test.coverage_branch = args['coverage_branch']
 
     if args['verbose'] >= VERBOSE_SHOWCASES:
         test.print_cases = True
@@ -158,6 +168,22 @@ def result_is_fail(result):
 def main():
     args = parse_args()
 
+    if args['coverage_min'] or args['coverage_branch']:
+        args['coverage_min'] = int(args['coverage_min'])
+        args['coverage'] = True
+
+    if args['coverage']:
+        import coverage
+        cov = coverage.Coverage(
+            data_file=os.path.join(args['path'], 'tests.coverage'),
+            data_suffix=False,
+            auto_data=False,
+            branch=args['coverage_branch']
+        )
+        cov.start()
+
+    coverage_files = [os.path.join(args['path'], 'tests.coverage')]
+
     tests = get_tests()
     if args['list_tests']:
         for testname in tests.keys():
@@ -179,6 +205,7 @@ def main():
         os.makedirs(args['path'])
 
     test_results = {}
+    anyfailures = False
 
     for test in args['test']:
         if args['verbose'] >= VERBOSE_SHOWTESTS:
@@ -189,9 +216,34 @@ def main():
         if args['verbose'] >= VERBOSE_SHOWTESTS:
             print(result_to_str(result))
 
+        anyfailures = anyfailures or result_is_fail(result)
+
         if args['fail_on_first_error'] and result_is_fail(result):
             break
 
+        coverage_files.append(os.path.join(args['path'],
+                              test,
+                              'installer.coverage'))
+        coverage_files.extend(glob.glob(
+            '%s/%s/lib/*/coverage.*' % (args['path'], test)))
+
+    if args['coverage']:
+        cov.stop()
+        cov.save()
+
+    if args['coverage'] and not anyfailures:
+        print('Collecting coverage reports...')
+
+        cov = coverage.Coverage(
+            data_file=os.path.join(args['path'], 'combined.coverage'),
+            branch=args['coverage_branch'])
+        cov.combine(data_paths=coverage_files)
+        cov.save()
+        cov_perc = cov.report(include='ipsilon/*')
+        cov.xml_report(
+            include='ipsilon/*',
+            outfile=os.path.join(args['path'], 'combined.coverage.xml'))
+
     if not args['no_overview']:
         print(args['results_header'])
         for test in test_results:
@@ -201,6 +253,12 @@ def main():
            for result in test_results.values()):
         sys.exit(1)
 
+    if (not anyfailures and args['coverage_min'] and
+            cov_perc < args['coverage_min']):
+        print('Test coverage was %d%%, less than required %d%%' %
+              (cov_perc, args['coverage_min']))
+        sys.exit(1)
+
 
 if __name__ == '__main__':
     main()

From 5aaaeda23bede79503e11c41a0b2e54cf55383a8 Mon Sep 17 00:00:00 2001
From: Patrick Uiterwijk <puiterwijk@redhat.com>
Date: Nov 20 2017 16:48:03 +0000
Subject: [PATCH 2/2] Run coverage tests with min coverage for Fedora27 containertest


Signed-off-by: Patrick Uiterwijk <puiterwijk@redhat.com>

---

diff --git a/Makefile b/Makefile
index 536361a..9faa3b3 100644
--- a/Makefile
+++ b/Makefile
@@ -170,7 +170,11 @@ containertest-fedora26: container-fedora26
 
 containertest-fedora27: container-fedora27
 	@echo "Starting Fedora 27 tests ..."
-	@docker run -v `pwd`:/code -t --rm ipsilon-fedora27
+	# This one is special, since it's the latest version, so supposed to support the most features.
+	# As such, we do coverage testing on this one.
+	@docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/make ipsilon-fedora27 testdeps lp-test unittests
+	# This "coverage-min" number should be increased everytime the coverage increases to make sure we don't regress.
+	@docker run -v `pwd`:/code -t --rm --entrypoint /usr/bin/bash ipsilon-fedora27 ./runtests --path=$(TESTDIR) --coverage --coverage-min=68
 	@echo "Fedora 27 passed"
 
 containertest-lint: container-centos7
diff --git a/tests/containers/Dockerfile-fedora b/tests/containers/Dockerfile-fedora
index c519795..f3de0e6 100644
--- a/tests/containers/Dockerfile-fedora
+++ b/tests/containers/Dockerfile-fedora
@@ -1 +1 @@
-RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib
+RUN yum install -y etcd python2-python-etcd dbus-python python2-ipalib python2-coverage

