From ca5ce5f3fec81d8ed23b4b31f411c289ed53dfc3 Mon Sep 17 00:00:00 2001 From: Yuxiang Zhu Date: Aug 21 2018 10:02:53 +0000 Subject: CI/CD: Move dev pipeline files into openshift/pipeines subdirectories This PR - moves the dev pipeline template into openshift/pipeines/templates. - adds template parameters to openshift/pipelines/jobs. - updates README.md accordingly. - splits inline Jenkinsfile to a seperated file. - modifies function test stage to use the waiverdb-integration-test pipeline job as a subroutine. --- diff --git a/openshift/README.md b/openshift/README.md index 8301f72..3268392 100644 --- a/openshift/README.md +++ b/openshift/README.md @@ -50,14 +50,14 @@ Then instantiate the template with default parameters (this will set up a pipeline to build the upstream repo and publish the result to upstream registries): ```bash - oc process --local -f ./waiverdb-dev-pipeline.yml | oc apply -f - + oc process --local -f pipelines/tempates/waiverdb-dev-template.yaml | oc apply -f - ``` Or you want to set up a pipeline for your own fork: ```bash - NAME=waiverdb-dev-pipeline - oc process --local -f ./waiverdb-dev-pipeline-template.yml \ + NAME=waiverdb-dev + oc process --local -f pipelines/tempates/waiverdb-dev-template.yaml \ -p NAME="$NAME" \ -p WAIVERDB_GIT_REPO="https://pagure.io/forks//waiverdb.git" \ -p WAIVERDB_GIT_REF="" \ @@ -105,14 +105,14 @@ Before running the pipeline, you need to build a container image for Jenkins sla This step should be repeated every time you change the Dockerfile for the Jenkins slave pods. ```bash - oc start-build waiverdb-dev-pipeline-jenkins-slave + oc start-build waiverdb-dev-jenkins-slave ``` #### Trigger A Pipeline Build To trigger a pipeline build, start the `waiverdb-dev-pipeline` BuildConfig with the Git commit ID or branch that you want to test against: ```bash - oc start-build waiverdb-dev-pipeline -e WAIVERDB_GIT_REF= + oc start-build waiverdb-dev -e WAIVERDB_GIT_REF= ``` You can go to the OpenShift Web console to check the details of the pipeline build. diff --git a/openshift/pipelines/jobs/waiverdb-dev.env b/openshift/pipelines/jobs/waiverdb-dev.env new file mode 100644 index 0000000..cd378b1 --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-dev.env @@ -0,0 +1 @@ +NAME=waiverdb-dev diff --git a/openshift/pipelines/jobs/waiverdb-dev.tmpl b/openshift/pipelines/jobs/waiverdb-dev.tmpl new file mode 100644 index 0000000..1c2538d --- /dev/null +++ b/openshift/pipelines/jobs/waiverdb-dev.tmpl @@ -0,0 +1 @@ +waiverdb-dev-template.yaml diff --git a/openshift/pipelines/templates/waiverdb-dev-template.yaml b/openshift/pipelines/templates/waiverdb-dev-template.yaml new file mode 100644 index 0000000..ac139ea --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-dev-template.yaml @@ -0,0 +1,211 @@ +# Template to produce a new WaiverDB dev CI/CD pipeline in OpenShift. +# +# Dev pipeline is a part of the WaiverDB Pipeline, covers the following steps: +# +# - Run Flake8 and Pylint checks +# - Run unit tests +# - Build Docs +# - Publish Docs +# - Build SRPM +# - Build RPM +# - Invoke Rpmlint +# - Build container +# - Run functional tests +# - Push container +# +# Required Jenkins Plugins: +# - Openshift Sync plugin +# - Openshift Client plugin +# - Kubernetes plugin +# - SSH Agent plugin +# - Timestamper plugin +# +--- +apiVersion: v1 +kind: Template +metadata: + name: waiverdb-dev-pipeline +parameters: +- name: NAME + displayName: Short unique identifier for the templated instances + description: This field is used to deploy multiple pipelines to one OpenShift project from this template. + required: true + value: waiverdb-dev +- name: WAIVERDB_GIT_REPO + displayName: WaiverDB Git repo URL + description: Default WaiverDB Git repo URL in which to run dev tests against + required: true + value: "https://pagure.io/waiverdb.git" +- name: WAIVERDB_GIT_REF + displayName: WaiverDB Git repo ref + description: Default WaiverDB Git repo ref in which to run dev tests against + required: true + value: master +- name: WAIVERDB_MAIN_BRANCH + displayName: Name of the main branch. + description: If WAIVERDB_MAIN_BRANCH equals WAIVERDB_GIT_REF, publishing steps will be automatically performed. + value: master + required: true +- name: JENKINS_AGENT_CLOUD_NAME + displayName: Name of OpenShift cloud in Jenkins master configuration + required: true + value: openshift +- name: JENKINS_AGENT_IMAGE + displayName: Container image for Jenkins slave pods + required: true + value: docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest +- name: PAGURE_DOC_REPO_NAME + displayName: namespace/project of Pagure doc repo for publishing docs + description: If not emptry and WAIVERDB_GIT_REF is master, docs will be published to the specified Pagure doc repo + required: false + value: waiverdb +- name: PAGURE_DOC_SECRET + displayName: Name of the OpenShift SSH secret for publishing docs to Pagure. + required: false + value: pagure-doc-secret +- name: WAIVERDB_DEV_IMAGE_DESTINATIONS + displayName: Comma seperated list of container repository/tag to which the built WaiverDB dev image will be pushed + description: OpenShift registries must be prefixed with 'atomic:' + required: false + value: "atomic:docker-registry.engineering.redhat.com/factory2/waiverdb:latest,quay.io/factory2/waiverdb:latest" +- name: CONTAINER_REGISTRY_CREDENTIALS + displayName: Secret name of container registries used for pulling and pushing images + value: factory2-pipeline-registry-credentials + required: false +- name: WAIVERDB_DEV_IMAGE_TAG + displayName: Tag name of the resulting container image for development environment + value: "latest" + required: true +- name: WAIVERDB_IMAGESTREAM_NAME + displayName: Name of ImageStream for WaiverDB container images + required: true + value: waiverdb +- name: WAIVERDB_IMAGESTREAM_NAMESPACE + displayName: Namespace of ImageStream for WaiverDB container images + required: false +- name: WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAME + displayName: Name of BuildConfig for running integration tests + required: true + value: waiverdb-integration-test +- name: WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE + displayName: Namespace of BuildConfig for running integration tests + required: false +- name: FORCE_PUBLISH_IMAGE + displayName: Whether to push the resulting image regardless of the Git branch + value: "false" + required: true +- name: FORCE_PUBLISH_DOCS + displayName: Whether to publish docs regardless of the Git branch + value: "false" + required: true +- name: TAG_INTO_IMAGESTREAM + displayName: Whether to tag the pushed image as dev + value: "true" + required: true +labels: + template: waiverdb-dev +objects: +- kind: "BuildConfig" + apiVersion: "v1" + metadata: + name: "${NAME}-jenkins-slave" + labels: + app: "${NAME}" + spec: + runPolicy: "Serial" + completionDeadlineSeconds: 1800 + strategy: + dockerStrategy: + forcePull: true + dockerfilePath: openshift/containers/jenkins-slave/Dockerfile + resources: + requests: + memory: "512Mi" + cpu: "300m" + limits: + memory: "768Mi" + cpu: "500m" + source: + git: + uri: "${WAIVERDB_GIT_REPO}" + ref: "${WAIVERDB_GIT_REF}" + output: + to: + kind: "DockerImage" + name: "${JENKINS_AGENT_IMAGE}" + pushSecret: + name: "${CONTAINER_REGISTRY_CREDENTIALS}" + +- kind: ServiceAccount + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave" + labels: + app: "${NAME}" + +- kind: RoleBinding + apiVersion: v1 + metadata: + name: "${NAME}-jenkins-slave_edit" + labels: + app: "${NAME}" + subjects: + - kind: ServiceAccount + name: "${NAME}-jenkins-slave" + roleRef: + name: edit + +- kind: "BuildConfig" + apiVersion: "v1" + metadata: + name: "${NAME}" + labels: + app: "${NAME}" + spec: + runPolicy: "Serial" # FIXME: Parallel is supported, but we have limited quota in UpShift. + completionDeadlineSeconds: 1800 + source: + git: + uri: "${WAIVERDB_GIT_REPO}" + ref: "${WAIVERDB_GIT_REF}" + strategy: + type: JenkinsPipeline + jenkinsPipelineStrategy: + env: + - name: "WAIVERDB_GIT_REPO" + value: "${WAIVERDB_GIT_REPO}" + - name: "WAIVERDB_GIT_REF" + value: "${WAIVERDB_GIT_REF}" + - name: "JENKINS_AGENT_CLOUD_NAME" + value: "${JENKINS_AGENT_CLOUD_NAME}" + - name: "JENKINS_AGENT_IMAGE" + value: "${JENKINS_AGENT_IMAGE}" + - name: "JENKINS_AGENT_SERVICE_ACCOUNT" + value: "${NAME}-jenkins-slave" + - name: "WAIVERDB_DEV_IMAGE_DESTINATIONS" + value: "${WAIVERDB_DEV_IMAGE_DESTINATIONS}" + - name: "CONTAINER_REGISTRY_CREDENTIALS" + value: "${CONTAINER_REGISTRY_CREDENTIALS}" + - name: "FORCE_PUBLISH_IMAGE" + value: "${FORCE_PUBLISH_IMAGE}" + - name: "TAG_INTO_IMAGESTREAM" + value: "${TAG_INTO_IMAGESTREAM}" + - name: "WAIVERDB_DEV_IMAGE_TAG" + value: "${WAIVERDB_DEV_IMAGE_TAG}" + - name: "WAIVERDB_IMAGESTREAM_NAME" + value: "${WAIVERDB_IMAGESTREAM_NAME}" + - name: "WAIVERDB_IMAGESTREAM_NAMESPACE" + value: "${WAIVERDB_IMAGESTREAM_NAMESPACE}" + - name: "FORCE_PUBLISH_DOCS" + value: "${FORCE_PUBLISH_DOCS}" + - name: "PAGURE_DOC_REPO_NAME" + value: "${PAGURE_DOC_REPO_NAME}" + - name: "PAGURE_DOC_SECRET" + value: "${PAGURE_DOC_SECRET}" + - name: "WAIVERDB_MAIN_BRANCH" + value: "${WAIVERDB_MAIN_BRANCH}" + - name: "WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAME" + value: "${WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAME}" + - name: "WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE" + value: "${WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE}" + jenkinsfilePath: openshift/pipelines/templates/waiverdb-dev.Jenkinsfile diff --git a/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile b/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile new file mode 100644 index 0000000..327a046 --- /dev/null +++ b/openshift/pipelines/templates/waiverdb-dev.Jenkinsfile @@ -0,0 +1,410 @@ +pipeline { + agent { + kubernetes { + cloud params.JENKINS_AGENT_CLOUD_NAME + label "jenkins-slave-${UUID.randomUUID().toString()}" + serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: "jenkins-${env.JOB_BASE_NAME}" + factory2-pipeline-kind: "waiverdb-dev-pipeline" + factory2-pipeline-build-number: "${env.BUILD_NUMBER}" + spec: + containers: + - name: jnlp + image: "${params.JENKINS_AGENT_IMAGE}" + imagePullPolicy: Always + tty: true + env: + - name: REGISTRY_CREDENTIALS + valueFrom: + secretKeyRef: + name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" + key: '.dockerconfigjson' + # Required by unit tests: Set up NSS Wrapper to generate a fake user name for the random UID assigned by OpenShift + - name: LD_PRELOAD + value: '/usr/lib64/libnss_wrapper.so' + - name: NSS_WRAPPER_PASSWD + value: '/tmp/passwd' + - name: NSS_WRAPPER_GROUP + value: '/etc/group' + volumeMounts: + - name: postgresql-socket + mountPath: /var/run/postgresql + resources: + requests: + memory: 768Mi + cpu: 300m + limits: + memory: 1Gi + cpu: 500m + - name: db + image: registry.access.redhat.com/rhscl/postgresql-95-rhel7:latest + imagePullPolicy: Always + env: + - name: POSTGRESQL_USER + value: waiverdb + - name: POSTGRESQL_PASSWORD + value: waiverdb + - name: POSTGRESQL_DATABASE + value: waiverdb + volumeMounts: + - name: postgresql-socket + mountPath: /var/run/postgresql + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 384Mi + cpu: 200m + volumes: + - name: postgresql-socket + emptyDir: {} + """ + } + } + options { + timestamps() + timeout(time: 30, unit: 'MINUTES') + } + environment { + PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() + PIPELINE_USERNAME = sh(returnStdout: true, script: 'id -un').trim() + } + stages { + stage('Prepare') { + steps { + script { + def scmVars = checkout([$class: 'GitSCM', + branches: [[name: params.WAIVERDB_GIT_REF]], + userRemoteConfigs: [[url: params.WAIVERDB_GIT_REPO]], + ]) + env.WAIVERDB_GIT_COMMIT_ID = scmVars.GIT_COMMIT + // Generate a version-release number for the target Git commit + def versions = sh(returnStdout: true, script: 'source ./version.sh && echo -en "$WAIVERDB_VERSION\n$WAIVERDB_CONTAINER_VERSION"').split('\n') + env.WAIVERDB_VERSION = versions[0] + env.WAIVERDB_CONTAINER_VERSION = versions[1] + env.TEMP_TAG = env.WAIVERDB_CONTAINER_VERSION + '-jenkins-' + currentBuild.id + } + sh 'cp conf/settings.py.example conf/settings.py' + } + } + stage('Run checks') { + failFast false + parallel { + stage('Invoke Flake8') { + steps { + sh 'flake8' + } + } + stage('Invoke Pylint') { + steps { + sh 'pylint-3 --reports=n waiverdb' + } + } + } + } + stage('Run unit tests') { + steps { + // wait for the test datebase to come up + sh 'wait-for-it -s -t 300 127.0.0.1:5432' + // create a database role + sh 'psql -h 127.0.0.1 -U "postgres" -q -d "waiverdb" -c "CREATE ROLE \"$PIPELINE_USERNAME\" WITH LOGIN SUPERUSER;"' + // run unit tests + sh 'py.test-3 -v --junitxml=junit-tests.xml tests' + } + post { + always { + junit 'junit-tests.xml' + } + } + } + stage('Build Artifacts') { + failFast false + parallel { + stage('Branch Docs') { + stages { + stage('Build Docs') { + steps { + sh 'make -C docs html' + } + post { + always { + archiveArtifacts artifacts: 'docs/_build/html/**' + } + } + } + stage('Publish Docs') { + when { + expression { + return "${params.PAGURE_DOC_REPO_NAME}" && (params.WAIVERDB_GIT_REF == "${params.WAIVERDB_MAIN_BRANCH}" || env.FORCE_PUBLISH_DOCS == "true") + } + } + steps { + sshagent (credentials: ["${env.PIPELINE_NAMESPACE}-${params.PAGURE_DOC_SECRET}"]) { + sh ''' + mkdir -p ~/.ssh/ + touch ~/.ssh/known_hosts + ssh-keygen -R pagure.io + echo 'pagure.io ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC198DWs0SQ3DX0ptu+8Wq6wnZMrXUCufN+wdSCtlyhHUeQ3q5B4Hgto1n2FMj752vToCfNTn9mWO7l2rNTrKeBsELpubl2jECHu4LqxkRVihu5UEzejfjiWNDN2jdXbYFY27GW9zymD7Gq3u+T/Mkp4lIcQKRoJaLobBmcVxrLPEEJMKI4AJY31jgxMTnxi7KcR+U5udQrZ3dzCn2BqUdiN5dMgckr4yNPjhl3emJeVJ/uhAJrEsgjzqxAb60smMO5/1By+yF85Wih4TnFtF4LwYYuxgqiNv72Xy4D/MGxCqkO/nH5eRNfcJ+AJFE7727F7Tnbo4xmAjilvRria/+l' >>~/.ssh/known_hosts + rm -rf docs-on-pagure + git clone ssh://git@pagure.io/docs/${PAGURE_DOC_REPO_NAME}.git docs-on-pagure + rm -rf docs-on-pagure/* + cp -r docs/_build/html/* docs-on-pagure/ + cd docs-on-pagure + git config user.name 'Pipeline Bot' + git config user.email "pipeline-bot@localhost.localdomain" + git add -A . + if [[ "$(git diff --cached --numstat | wc -l)" -eq 0 ]] ; then + exit 0 # No changes, nothing to commit + fi + git commit -m 'Automatic commit of docs built by Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER}' + git push origin master + ''' + } + } + } + } + } + stage('Build SRPM') { + steps { + sh './rpmbuild.sh -bs' + } + post { + success { + archiveArtifacts artifacts: 'rpmbuild-output/*.src.rpm' + } + } + } + stage('Branch RPM') { + stages { + stage('Build RPM') { + steps { + sh './rpmbuild.sh -bb' + } + post { + success { + archiveArtifacts artifacts: 'rpmbuild-output/*/*.rpm' + } + } + } + stage('Invoke Rpmlint') { + steps { + sh 'rpmlint -f rpmlint-config.py rpmbuild-output/*/*.rpm' + } + } + } + } + } + } + stage('Build container') { + environment { + BUILDCONFIG_INSTANCE_ID = "waiverdb-container-build-${currentBuild.id}" + } + steps { + script { + openshift.withCluster() { + // OpenShift BuildConfig doesn't support specifying a tag name at build time. + // We have to create a new BuildConfig for each container build. + // Create a BuildConfig from a seperated Template. + echo 'Creating a BuildConfig for container build...' + def template = readYaml file: 'openshift/waiverdb-container-template.yml' + def processed = openshift.process(template, + "-p", "NAME=${env.BUILDCONFIG_INSTANCE_ID}", + '-p', "WAIVERDB_GIT_REPO=${params.WAIVERDB_GIT_REPO}", + '-p', "WAIVERDB_GIT_REF=${env.WAIVERDB_GIT_COMMIT_ID}", + '-p', "WAIVERDB_IMAGE_TAG=${env.TEMP_TAG}", + '-p', "WAIVERDB_VERSION=${env.WAIVERDB_VERSION}", + '-p', "WAIVERDB_IMAGESTREAM_NAME=${params.WAIVERDB_IMAGESTREAM_NAME}", + '-p', "WAIVERDB_IMAGESTREAM_NAMESPACE=${params.WAIVERDB_IMAGESTREAM_NAMESPACE}", + ) + def created = openshift.apply(processed) + def bc = created.narrow('bc') + echo 'Starting a container build from the created BuildConfig...' + buildSelector = bc.startBuild() + // `buildSelector.logs()` can be dumb when the OpenShift Build is not started. + // Let's wait for it to be started or completed (failed). + echo 'Waiting for the container build to be started...' + timeout(5) { // 5 min + buildSelector.watch { + return !(it.object().status.phase in ["New", "Pending", "Unknown"]) + } + } + echo 'Following container build logs...' + // This function sometimes hangs infinitely. + // Not sure it is a problem of OpenShift Jenkins Client plugin + // or OpenShift. + // FIXME: logs() step may fail with unknown reasons. + timeout(time: 15, activity: false) { + buildSelector.logs('-f') + } + // Ensure the build is stopped + echo 'Waiting for the container build to be fully stopped...' + timeout(5) { // 5 min + buildSelector.watch { + return it.object().status.phase != "Running" + } + } + // Assert build result + def ocpBuild = buildSelector.object() + if (ocpBuild.status.phase != "Complete") { + error("Failed to build container image for ${env.TEMP_TAG}, .status.phase=${ocpBuild.status.phase}.") + } + echo 'Container build is complete.' + env.RESULTING_IMAGE_REF = ocpBuild.status.outputDockerImageReference + env.RESULTING_IMAGE_DIGEST = ocpBuild.status.output.to.imageDigest + def imagestream= created.narrow('is').object() + env.RESULTING_IMAGE_REPO = imagestream.status.dockerImageRepository + env.RESULTING_TAG = env.TEMP_TAG + } + } + } + post { + cleanup { + script { + openshift.withCluster() { + echo 'Tearing down...' + openshift.selector('bc', [ + 'app': env.BUILDCONFIG_INSTANCE_ID, + 'template': 'waiverdb-container-template', + ]).delete() + } + } + } + } + } + stage('Run functional tests') { + steps { + script { + openshift.withCluster() { + openshift.withProject(params.WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAMESPACE) { + def testBcSelector = openshift.selector('bc', params.WAIVERDB_INTEGRATION_TEST_BUILD_CONFIG_NAME) + echo 'Starting a functional test for the built container image...' + def buildSelector = testBcSelector.startBuild( + '-e', "IMAGE=${env.RESULTING_IMAGE_REPO}:${env.RESULTING_TAG}", + '-e', "WAIVERDB_GIT_REPO=${params.WAIVERDB_GIT_REPO}", + '-e', "WAIVERDB_GIT_REF=${env.WAIVERDB_GIT_COMMIT_ID}", + ) + timeout(5) { // 5 min + buildSelector.watch { + return !(it.object().status.phase in ["New", "Pending", "Unknown"]) + } + } + echo 'Following functional test logs...' + // This function sometimes hangs infinitely. Not sure it is a problem of OpenShift Jenkins Client plugin or OpenShift. + timeout(time: 15) { + buildSelector.logs('-f') + } + echo 'Waiting for the integration test to be fully stopped...' + timeout(5) { // 5 min + buildSelector.watch { + return it.object().status.phase != "Running" + } + } + // Assert build result + def ocpBuild = buildSelector.object() + if (ocpBuild.status.phase != "Complete") { + error("Functional test failed for image ${env.TEMP_TAG}, .status.phase=${ocpBuild.status.phase}.") + } + echo 'Functional test is PASSED.' + } + } + } + } + } + stage('Push container') { + when { + expression { + return params.FORCE_PUBLISH_IMAGE == 'true' || + params.WAIVERDB_GIT_REF == params.WAIVERDB_MAIN_BRANCH + } + } + steps { + script { + def destinations = env.WAIVERDB_DEV_IMAGE_DESTINATIONS ? + env.WAIVERDB_DEV_IMAGE_DESTINATIONS.split(',') : [] + openshift.withCluster() { + def sourceImage = env.RESULTING_IMAGE_REPO + ":" + env.RESULTING_TAG + if (env.REGISTRY_CREDENTIALS) { + dir ("${env.HOME}/.docker") { + writeFile file:'config.json', text: env.REGISTRY_CREDENTIALS + } + } + // pull the built image from imagestream + echo "Pulling container from ${sourceImage}..." + def registryToken = readFile(file: '/var/run/secrets/kubernetes.io/serviceaccount/token') + withEnv(["SOURCE_IMAGE_REF=${sourceImage}", "TOKEN=${registryToken}"]) { + sh '''set -e +x # hide the token from Jenkins console + mkdir -p _build + skopeo copy \ + --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ \ + --src-creds=serviceaccount:"$TOKEN" \ + docker://"$SOURCE_IMAGE_REF" dir:_build/waiverdb_container + ''' + } + // push to registries + def pushTasks = destinations.collectEntries { + ["Pushing ${it}" : { + def dest = it + // Only docker and atomic registries are allowed + if (!it.startsWith('atomic:') && !it.startsWith('docker://')) { + dest = 'docker://' + it + } + echo "Pushing container to ${dest}..." + withEnv(["DEST_IMAGE_REF=${dest}"]) { + /* Pushes to the internal registry can sometimes randomly fail + * with "unknown blob" due to a known issue with the registry + * storage configuration. So we retry up to 5 times. */ + retry(5) { + sh 'skopeo copy dir:_build/waiverdb_container "$DEST_IMAGE_REF"' + } + } + }] + } + parallel pushTasks + } + } + } + } + stage('Tag into image stream') { + when { + expression { + return "${params.WAIVERDB_DEV_IMAGE_TAG}" && params.TAG_INTO_IMAGESTREAM == "true" && + (params.FORCE_PUBLISH_IMAGE == 'true' || params.WAIVERDB_GIT_REF == params.WAIVERDB_MAIN_BRANCH) + } + } + steps { + script { + openshift.withCluster() { + openshift.withProject("${params.WAIVERDB_IMAGESTREAM_NAMESPACE}") { + def sourceRef = "${params.WAIVERDB_IMAGESTREAM_NAME}:${env.RESULTING_TAG}" + def destRef = "${params.WAIVERDB_IMAGESTREAM_NAME}:${params.WAIVERDB_DEV_IMAGE_TAG}" + echo "Tagging ${sourceRef} as ${destRef}..." + openshift.tag("${sourceRef}", "${destRef}") + } + } + } + } + } + } + post { + cleanup { + script { + if (env.RESULTING_TAG) { + echo "Removing tag ${env.RESULTING_TAG} from the ImageStream..." + openshift.withCluster() { + openshift.withProject("${params.WAIVERDB_IMAGESTREAM_NAMESPACE}") { + openshift.tag("${params.WAIVERDB_IMAGESTREAM_NAME}:${env.RESULTING_TAG}", + "-d") + } + } + } + } + } + } +} diff --git a/openshift/waiverdb-dev-pipeline-template.yml b/openshift/waiverdb-dev-pipeline-template.yml deleted file mode 100644 index 49241e2..0000000 --- a/openshift/waiverdb-dev-pipeline-template.yml +++ /dev/null @@ -1,629 +0,0 @@ -# Template to produce a new WaiverDB dev CI/CD pipeline in OpenShift. -# -# Dev pipeline is a part of the WaiverDB Pipeline, covers the following steps: -# -# - Run Flake8 and Pylint checks -# - Run unit tests -# - Build Docs -# - Publish Docs -# - Build SRPM -# - Build RPM -# - Invoke Rpmlint -# - Build container -# - Run functional tests -# - Push container -# -# Required Jenkins Plugins: -# - Openshift Sync plugin -# - Openshift Client plugin -# - Kubernetes plugin -# - SSH Agent plugin -# - Timestamper plugin -# ---- -apiVersion: v1 -kind: Template -metadata: - name: waiverdb-dev-pipeline -parameters: -- name: NAME - displayName: Short unique identifier for the templated instances - description: This field is used to deploy multiple pipelines to one OpenShift project from this template. - required: true - value: waiverdb-dev-pipeline -- name: WAIVERDB_GIT_REPO - displayName: WaiverDB Git repo URL - description: Default WaiverDB Git repo URL in which to run dev tests against - required: true - value: "https://pagure.io/waiverdb.git" -- name: WAIVERDB_GIT_REF - displayName: WaiverDB Git repo ref - description: Default WaiverDB Git repo ref in which to run dev tests against - required: true - value: master -- name: WAIVERDB_MAIN_BRANCH - displayName: Name of the main branch. - description: If WAIVERDB_MAIN_BRANCH equals WAIVERDB_GIT_REF, publishing steps will be automatically performed. - value: master - required: true -- name: JENKINS_AGENT_CLOUD_NAME - displayName: Name of OpenShift cloud in Jenkins master configuration - required: true - value: openshift -- name: JENKINS_AGENT_IMAGE - displayName: Container image for Jenkins slave pods - required: true - value: docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest -- name: PAGURE_DOC_REPO_NAME - displayName: namespace/project of Pagure doc repo for publishing docs - description: If not emptry and WAIVERDB_GIT_REF is master, docs will be published to the specified Pagure doc repo - required: false - value: waiverdb -- name: PAGURE_DOC_SECRET - displayName: Name of the OpenShift SSH secret for publishing docs to Pagure. - required: false - value: pagure-waiverdb-deploy-key -- name: WAIVERDB_DEV_IMAGE_DESTINATIONS - displayName: Comma seperated list of container repository/tag to which the built WaiverDB dev image will be pushed - description: OpenShift registries must be prefixed with 'atomic:' - required: false - value: "atomic:docker-registry.engineering.redhat.com/factory2/waiverdb:latest,quay.io/factory2/waiverdb:latest" -- name: CONTAINER_REGISTRY_CREDENTIALS - displayName: Secret name of container registries used for pulling and pushing images - value: factory2-pipeline-registry-credentials - required: false -- name: WAIVERDB_DEV_IMAGE_TAG - displayName: Tag name of the resulting container image for development environment - value: "latest" - required: true -- name: WAIVERDB_IMAGESTREAM_NAME - displayName: Name of ImageStream for WaiverDB container images - required: true - value: waiverdb -- name: WAIVERDB_IMAGESTREAM_NAMESPACE - displayName: Namespace of ImageStream for WaiverDB container images - required: false -- name: FORCE_PUBLISH_IMAGE - displayName: Whether to push the resulting image regardless of the Git branch - value: "false" - required: true -- name: FORCE_PUBLISH_DOCS - displayName: Whether to publish docs regardless of the Git branch - value: "false" - required: true -- name: TAG_INTO_IMAGESTREAM - displayName: Whether to tag the pushed image as dev - value: "true" - required: true -labels: - template: waiverdb-dev-pipeline -objects: -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" - spec: - runPolicy: "Serial" - completionDeadlineSeconds: 1800 - strategy: - dockerStrategy: - forcePull: true - dockerfilePath: openshift/containers/jenkins-slave/Dockerfile - resources: - requests: - memory: "512Mi" - cpu: "300m" - limits: - memory: "768Mi" - cpu: "500m" - source: - git: - uri: "${WAIVERDB_GIT_REPO}" - ref: "${WAIVERDB_GIT_REF}" - output: - to: - kind: "DockerImage" - name: "${JENKINS_AGENT_IMAGE}" - pushSecret: - name: "${CONTAINER_REGISTRY_CREDENTIALS}" - -- kind: ServiceAccount - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave" - labels: - app: "${NAME}" - -- kind: RoleBinding - apiVersion: v1 - metadata: - name: "${NAME}-jenkins-slave_edit" - labels: - app: "${NAME}" - subjects: - - kind: ServiceAccount - name: "${NAME}-jenkins-slave" - roleRef: - name: edit - -- kind: "BuildConfig" - apiVersion: "v1" - metadata: - name: "${NAME}" - labels: - app: "${NAME}" - spec: - runPolicy: "Serial" # FIXME: Parallel is supported, but we have limited quota in UpShift. - completionDeadlineSeconds: 1800 - strategy: - type: JenkinsPipeline - source: - type: None - jenkinsPipelineStrategy: - env: - - name: "WAIVERDB_GIT_REPO" - value: "${WAIVERDB_GIT_REPO}" - - name: "WAIVERDB_GIT_REF" - value: "${WAIVERDB_GIT_REF}" - - name: "JENKINS_AGENT_CLOUD_NAME" - value: "${JENKINS_AGENT_CLOUD_NAME}" - - name: "JENKINS_AGENT_SERVICE_ACCOUNT" - value: "${NAME}-jenkins-slave" - - name: "WAIVERDB_DEV_IMAGE_DESTINATIONS" - value: "${WAIVERDB_DEV_IMAGE_DESTINATIONS}" - - name: "FORCE_PUBLISH_IMAGE" - value: "${FORCE_PUBLISH_IMAGE}" - - name: "FORCE_PUBLISH_DOCS" - value: "${FORCE_PUBLISH_DOCS}" - - name: "TAG_INTO_IMAGESTREAM" - value: "${TAG_INTO_IMAGESTREAM}" - jenkinsfile: |- - pipeline { - agent { - kubernetes { - cloud params.JENKINS_AGENT_CLOUD_NAME - label "jenkins-slave-${UUID.randomUUID().toString()}" - serviceAccount params.JENKINS_AGENT_SERVICE_ACCOUNT - defaultContainer 'jnlp' - yaml """ - apiVersion: v1 - kind: Pod - metadata: - labels: - app: "${NAME}" - factory2-pipeline-kind: "waiverdb-dev-pipeline" - factory2-pipeline-build-number: "${env.BUILD_NUMBER}" - spec: - containers: - - name: jnlp - image: "${JENKINS_AGENT_IMAGE}" - imagePullPolicy: Always - tty: true - env: - - name: REGISTRY_CREDENTIALS - valueFrom: - secretKeyRef: - name: "${CONTAINER_REGISTRY_CREDENTIALS}" - key: '.dockerconfigjson' - # Set up NSS Wrapper to generate a fake user name for the random UID assigned by OpenShift - - name: LD_PRELOAD - value: '/usr/lib64/libnss_wrapper.so' - - name: NSS_WRAPPER_PASSWD - value: '/tmp/passwd' - - name: NSS_WRAPPER_GROUP - value: '/etc/group' - imagePullSecrets: - - "${CONTAINER_REGISTRY_CREDENTIALS}" - volumeMounts: - - name: postgresql-socket - mountPath: /var/run/postgresql - resources: - requests: - memory: 768Mi - cpu: 300m - limits: - memory: 1Gi - cpu: 500m - - name: db - image: registry.access.redhat.com/rhscl/postgresql-95-rhel7:latest - imagePullPolicy: Always - imagePullSecrets: - - "${CONTAINER_REGISTRY_CREDENTIALS}" - env: - - name: POSTGRESQL_USER - value: waiverdb - - name: POSTGRESQL_PASSWORD - value: waiverdb - - name: POSTGRESQL_DATABASE - value: waiverdb - volumeMounts: - - name: postgresql-socket - mountPath: /var/run/postgresql - resources: - requests: - memory: 256Mi - cpu: 100m - limits: - memory: 384Mi - cpu: 200m - volumes: - - name: postgresql-socket - emptyDir: {} - """ - } - } - options { - timestamps() - timeout(time: 30, unit: 'MINUTES') - } - stages { - stage('Prepare') { - steps { - checkout([$class: 'GitSCM', - branches: [[name: params.WAIVERDB_GIT_REF]], - userRemoteConfigs: [[url: params.WAIVERDB_GIT_REPO]], - ]) - script { - env.WAIVERDB_GIT_COMMIT_ID = sh(returnStdout: true, script: 'git rev-parse HEAD').trim() - // Generate a version-release number for the target Git commit - def versions = sh(returnStdout: true, script: 'source ./version.sh && echo -en "$WAIVERDB_VERSION\n$WAIVERDB_CONTAINER_VERSION"').split('\n') - env.WAIVERDB_VERSION = versions[0] - env.WAIVERDB_CONTAINER_VERSION = versions[1] - env.TEMP_TAG = env.WAIVERDB_CONTAINER_VERSION + '-jenkins-' + currentBuild.id - env.PIPELINE_NAMESPACE = readFile('/run/secrets/kubernetes.io/serviceaccount/namespace').trim() - env.PIPELINE_USERNAME = sh(returnStdout: true, script: 'id -un').trim() - } - sh 'cp conf/settings.py.example conf/settings.py' - } - } - stage('Run checks') { - failFast false - parallel { - stage('Invoke Flake8') { - steps { - sh 'flake8' - } - } - stage('Invoke Pylint') { - steps { - sh 'pylint-3 --reports=n waiverdb' - } - } - } - } - stage('Run unit tests') { - steps { - // wait for the test datebase to come up - sh 'wait-for-it -s -t 300 127.0.0.1:5432' - // create a database role - sh 'psql -h 127.0.0.1 -U "postgres" -q -d "waiverdb" -c "CREATE ROLE \"$PIPELINE_USERNAME\" WITH LOGIN SUPERUSER;"' - // run unit tests - sh 'py.test-3 -v --junitxml=junit-tests.xml tests' - } - post { - always { - junit 'junit-tests.xml' - } - } - } - stage('Build Artifacts') { - failFast false - parallel { - stage('Branch Docs') { - stages { - stage('Build Docs') { - steps { - sh 'make -C docs html' - } - post { - always { - archiveArtifacts artifacts: 'docs/_build/html/**' - } - } - } - stage('Publish Docs') { - when { - expression { - return "${PAGURE_DOC_REPO_NAME}" && (params.WAIVERDB_GIT_REF == "${WAIVERDB_MAIN_BRANCH}" || env.FORCE_PUBLISH_DOCS == "true") - } - } - steps { - sshagent (credentials: [env.PIPELINE_NAMESPACE + '-${PAGURE_DOC_SECRET}']) { - sh ''' - mkdir -p ~/.ssh/ - touch ~/.ssh/known_hosts - ssh-keygen -R pagure.io - echo 'pagure.io ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC198DWs0SQ3DX0ptu+8Wq6wnZMrXUCufN+wdSCtlyhHUeQ3q5B4Hgto1n2FMj752vToCfNTn9mWO7l2rNTrKeBsELpubl2jECHu4LqxkRVihu5UEzejfjiWNDN2jdXbYFY27GW9zymD7Gq3u+T/Mkp4lIcQKRoJaLobBmcVxrLPEEJMKI4AJY31jgxMTnxi7KcR+U5udQrZ3dzCn2BqUdiN5dMgckr4yNPjhl3emJeVJ/uhAJrEsgjzqxAb60smMO5/1By+yF85Wih4TnFtF4LwYYuxgqiNv72Xy4D/MGxCqkO/nH5eRNfcJ+AJFE7727F7Tnbo4xmAjilvRria/+l' >>~/.ssh/known_hosts - rm -rf docs-on-pagure - git clone ssh://git@pagure.io/docs/${PAGURE_DOC_REPO_NAME}.git docs-on-pagure - rm -rf docs-on-pagure/* - cp -r docs/_build/html/* docs-on-pagure/ - cd docs-on-pagure - git config user.name 'Pipeline Bot' - git config user.email "pipeline-bot@localhost.localdomain" - git add -A . - if [[ "$(git diff --cached --numstat | wc -l)" -eq 0 ]] ; then - exit 0 # No changes, nothing to commit - fi - git commit -m 'Automatic commit of docs built by Jenkins job ${env.JOB_NAME} #${env.BUILD_NUMBER}' - git push origin master - ''' - } - } - } - } - } - stage('Build SRPM') { - steps { - sh './rpmbuild.sh -bs' - } - post { - success { - archiveArtifacts artifacts: 'rpmbuild-output/*.src.rpm' - } - } - } - stage('Branch RPM') { - stages { - stage('Build RPM') { - steps { - sh './rpmbuild.sh -bb' - } - post { - success { - archiveArtifacts artifacts: 'rpmbuild-output/*/*.rpm' - } - } - } - stage('Invoke Rpmlint') { - steps { - sh 'rpmlint -f rpmlint-config.py rpmbuild-output/*/*.rpm' - } - } - } - } - } - } - stage('Build container') { - //options { - // timeout(time: 15, unit: 'MINUTES') - //} - environment { - BUILDCONFIG_INSTANCE_ID = "waiverdb-container-build-${currentBuild.id}" - } - steps { - script { - openshift.withCluster() { - // OpenShift BuildConfig doesn't support specifying a tag name at build time. - // We have to create a new BuildConfig for each container build. - // Create a BuildConfig from a seperated Template. - echo 'Creating a BuildConfig...' - def template = readYaml file: 'openshift/waiverdb-container-template.yml' - def processed = openshift.process(template, - "-p", "NAME=" + env.BUILDCONFIG_INSTANCE_ID, - '-p', 'WAIVERDB_GIT_REPO=' + params.WAIVERDB_GIT_REPO, - '-p', 'WAIVERDB_GIT_REF=' + env.WAIVERDB_GIT_COMMIT_ID, - '-p', 'WAIVERDB_IMAGE_TAG=' + env.TEMP_TAG, - '-p', 'WAIVERDB_VERSION=' + env.WAIVERDB_VERSION, - '-p', 'WAIVERDB_IMAGESTREAM_NAME=${WAIVERDB_IMAGESTREAM_NAME}', - '-p', 'WAIVERDB_IMAGESTREAM_NAMESPACE=${WAIVERDB_IMAGESTREAM_NAMESPACE}', - ) - def created = openshift.apply(processed) - def bc = created.narrow('bc') - echo 'Starting a container build from the created BuildConfig...' - buildSelector = bc.startBuild() - // `buildSelector.logs()` can be dumb when the OpenShift Build is not started. - // Let's wait for it to be started or completed (failed). - echo 'Waiting for the container build to be started...' - timeout(5) { // 5 min - buildSelector.watch { - return !(it.object().status.phase in ["New", "Pending"]) - } - } - echo 'Following container build logs...' - // This function sometimes hangs infinitely. - // Not sure it is a problem of OpenShift Jenkins Client plugin - // or OpenShift. - // FIXME: logs() step may fail with unknown reasons. - timeout(time: 15, activity: false) { - buildSelector.logs('-f') - } - // Ensure the build is stopped - echo 'Waiting for the container build to be fully stopped...' - timeout(5) { // 5 min - buildSelector.watch { - return it.object().status.phase != "Running" - } - } - // Assert build result - def ocpBuild = buildSelector.object() - if (ocpBuild.status.phase != "Complete") { - error("Failed to build container image for ${env.TEMP_TAG}, .status.phase=${ocpBuild.status.phase}.") - } - echo 'Container build is complete.' - env.RESULTING_IMAGE_REF = ocpBuild.status.outputDockerImageReference - env.RESULTING_IMAGE_DIGEST = ocpBuild.status.output.to.imageDigest - def imagestream= created.narrow('is').object() - env.RESULTING_IMAGE_REPO = imagestream.status.dockerImageRepository - env.RESULTING_TAG = env.TEMP_TAG - } - } - } - post { - cleanup { - script { - openshift.withCluster() { - echo 'Tearing down...' - openshift.selector('bc', [ - 'app': env.BUILDCONFIG_INSTANCE_ID, - 'template': 'waiverdb-container-template', - ]).delete() - } - } - } - } - } - stage('Run functional tests') { - environment { - // Jenkins BUILD_TAG could be too long (> 63 characters) for OpenShift to consume - TEST_ID = "jenkins-${currentBuild.id}" - ENVIRONMENT_LABEL = "test-${env.TEST_ID}" - } - steps { - script { - openshift.withCluster() { - def template = readYaml file: 'openshift/waiverdb-test-template.yaml' - def webPodReplicas = 1 // The current quota in UpShift is agressively limited - def models = openshift.process(template, - '-p', "TEST_ID=${env.TEST_ID}", - '-p', "WAIVERDB_APP_IMAGE_REPO=${env.RESULTING_IMAGE_REPO}", - '-p', "WAIVERDB_APP_VERSION=${env.RESULTING_TAG}", - '-p', "WAIVERDB_REPLICAS=${webPodReplicas}", - ) - def objects = openshift.apply(models) - echo "Waiting for test pods with label environment=${env.ENVIRONMENT_LABEL} to become Ready" - //def rm = dcSelector.rollout() - def dcs = openshift.selector('dc', ['environment': env.ENVIRONMENT_LABEL]) - def rm = dcs.rollout() - def pods = openshift.selector('pods', ['environment': env.ENVIRONMENT_LABEL]) - timeout(15) { - pods.untilEach(webPodReplicas + 1) { - def pod = it.object() - if (pod.status.phase in ["Pending", "Unknown"]) { - return false - } - if (pod.status.phase == "Running") { - for (cond in pod.status.conditions) { - if (cond.type == 'Ready' && cond.status == 'True') { - return true - } - } - return false - } - error("Test pod ${pod.metadata.name} is not running. Current phase is ${pod.status.phase}.") - } - } - // Run functional tests - def route_hostname = objects.narrow('route').object().spec.host - echo "Running tests against https://${route_hostname}/" - withEnv(["WAIVERDB_TEST_URL=https://${route_hostname}/"]) { - sh 'py.test-3 -v --junitxml=junit-functional-tests.xml functional-tests/' - } - } - } - } - post { - always { - script { - junit 'junit-functional-tests.xml' - openshift.withCluster() { - /* Extract logs for debugging purposes */ - openshift.selector('deploy,pods', ['environment': env.ENVIRONMENT_LABEL]).logs() - } - } - } - cleanup { - script { - openshift.withCluster() { - /* Tear down everything we just created */ - echo "Tearing down test resources..." - openshift.selector('dc,deploy,configmap,secret,svc,route', - ['environment': env.ENVIRONMENT_LABEL]).delete() - } - } - } - } - } - stage('Push container') { - when { - expression { - return params.FORCE_PUBLISH_IMAGE == 'true' || - params.WAIVERDB_GIT_REF == "${WAIVERDB_MAIN_BRANCH}" - } - } - steps { - script { - def destinations = env.WAIVERDB_DEV_IMAGE_DESTINATIONS ? - env.WAIVERDB_DEV_IMAGE_DESTINATIONS.split(',') : [] - openshift.withCluster() { - def sourceImage = env.RESULTING_IMAGE_REPO + ":" + env.RESULTING_TAG - if (env.REGISTRY_CREDENTIALS) { - dir ("${env.HOME}/.docker") { - writeFile file:'config.json', text: env.REGISTRY_CREDENTIALS - } - } - // pull the built image from imagestream - echo "Pulling container from ${sourceImage}..." - def registryToken = readFile(file: '/var/run/secrets/kubernetes.io/serviceaccount/token') - withEnv(["SOURCE_IMAGE_REF=${sourceImage}", "TOKEN=${registryToken}"]) { - sh '''set -e +x # hide the token from Jenkins console - mkdir -p _build - skopeo copy \ - --src-cert-dir=/var/run/secrets/kubernetes.io/serviceaccount/ \ - --src-creds=serviceaccount:"$TOKEN" \ - docker://"$SOURCE_IMAGE_REF" dir:_build/waiverdb_container - ''' - } - // push to registries - def pushTasks = destinations.collectEntries { - ["Pushing ${it}" : { - def dest = it - // Only docker and atomic registries are allowed - if (!it.startsWith('atomic:') && !it.startsWith('docker://')) { - dest = 'docker://' + it - } - echo "Pushing container to ${dest}..." - withEnv(["DEST_IMAGE_REF=${dest}"]) { - /* Pushes to the internal registry can sometimes randomly fail - * with "unknown blob" due to a known issue with the registry - * storage configuration. So we retry up to 5 times. */ - retry(5) { - sh 'skopeo copy dir:_build/waiverdb_container "$DEST_IMAGE_REF"' - } - } - }] - } - parallel pushTasks - } - } - } - } - stage('Tag into image stream') { - when { - expression { - return "${WAIVERDB_DEV_IMAGE_TAG}" && params.TAG_INTO_IMAGESTREAM == "true" && - (params.FORCE_PUBLISH_IMAGE == 'true' || params.WAIVERDB_GIT_REF == "${WAIVERDB_MAIN_BRANCH}") - } - } - steps { - script { - openshift.withCluster() { - openshift.withProject("${WAIVERDB_IMAGESTREAM_NAMESPACE}") { - def sourceRef = "${WAIVERDB_IMAGESTREAM_NAME}:${env.RESULTING_TAG}" - def destRef = "${WAIVERDB_IMAGESTREAM_NAME}:${WAIVERDB_DEV_IMAGE_TAG}" - echo "Tagging ${sourceRef} as ${destRef}..." - openshift.tag("${sourceRef}", "${destRef}") - } - } - } - } - } - } - post { - cleanup { - script { - if (env.RESULTING_TAG) { - echo "Removing tag ${env.RESULTING_TAG} from the ImageStream..." - openshift.withCluster() { - openshift.withProject("${WAIVERDB_IMAGESTREAM_NAMESPACE}") { - openshift.tag("${WAIVERDB_IMAGESTREAM_NAME}:${env.RESULTING_TAG}", - "-d") - } - } - } - } - } - } - }