From 86abb7c468c035e8b98a3ea47d67979f71e2259e Mon Sep 17 00:00:00 2001 From: Michal Kovarik Date: Sep 27 2019 06:39:23 +0000 Subject: [PATCH 1/3] Add global configuration for project Use global.env file instead of setting global variables for each job. Job environment file should contain configuration of template and should be static. File global.env contains variables which are used by multiple jobs. Typical content of global.env are URLs. Varibles priority: 1) use variable in job env file 2) use variable in global config if template contains it 3) use default variable in yaml template Installation of jobs using 'make install' will fail with error code in case of issue with configuration and other jobs are not applied. This behavior can be disabled by FAIL_ON_ERROR=false --- diff --git a/openshift/pipelines/Makefile b/openshift/pipelines/Makefile index 993653c..512ff45 100644 --- a/openshift/pipelines/Makefile +++ b/openshift/pipelines/Makefile @@ -4,6 +4,7 @@ JOBS_DIR:=jobs TEMPLATES_DIR:=templates JOB_PARAM_FILES:=$(wildcard $(JOBS_DIR)/*.env) JOBS:=$(patsubst $(JOBS_DIR)/%.env,%,$(JOB_PARAM_FILES)) +FAIL_ON_ERROR=true OC_CMD=$(OC) $(OCFLAGS) @@ -19,12 +20,22 @@ help: @echo -e "\tOC\t\tUse this oc command" @echo -e "\tOCFLAGS\t\tOptions to append to the oc command arguments" install: - @for job in $(JOBS); do \ - echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \ + @global_params=$$(cat global.env);\ + for job in $(JOBS); do \ + echo "[PIPELINE] Updating pipeline job \"$${job}\"..." ; \ template_file=$$(cat ./$(JOBS_DIR)/$${job}.tmpl); \ - $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \ - --param-file ./$(JOBS_DIR)/$${job}.env | $(OC_CMD) apply -f -; \ - echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \ + template_params=$$(oc process --local --parameters -f ./$(TEMPLATES_DIR)/$${template_file} | tail -n+2 | awk '{print $$1}'); \ + add_param=""; \ + for global_param in $${global_params}; do \ + global_param_name=$$(echo $${global_param} | cut -f1 -d=); \ + if ! grep -q "^$${global_param_name}=" $(JOBS_DIR)/$${job}.env && echo "$${template_params}" | grep -q "^$${global_param_name}$$"; then \ + add_param="$${add_param} --param=$${global_param}"; \ + fi; \ + done; \ + $(OC_CMD) process --local -f ./$(TEMPLATES_DIR)/$${template_file} \ + --param-file ./$(JOBS_DIR)/$${job}.env $${add_param} | $(OC_CMD) apply -f - || \ + { [ "$(FAIL_ON_ERROR)" == "true" ] && { echo "[PIPELINE] Pipeline job \"$${job}\" update failed"; exit 1; };}; \ + echo "[PIPELINE] Pipeline job \"$${job}\" updated" ; \ done uninstall: @for job in $(JOBS); do \ diff --git a/openshift/pipelines/global.env b/openshift/pipelines/global.env new file mode 100644 index 0000000..dcda3ea --- /dev/null +++ b/openshift/pipelines/global.env @@ -0,0 +1,8 @@ +WAIVERDB_GIT_REPO=https://pagure.io/waiverdb.git +PAGURE_REPO_NAME=waiverdb +PAGURE_REPO_IS_FORK=false +WAIVERDB_DEV_IMAGE_DESTINATIONS=quay.io/factory2/waiverdb +JENKINS_AGENT_IMAGE=docker-registry.engineering.redhat.com/factory2/waiverdb-jenkins-slave:latest +MAIL_ADDRESS=pnt-factory2-devel@redhat.com +SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb +BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i diff --git a/openshift/pipelines/jobs/waiverdb-dev-integration-test.env b/openshift/pipelines/jobs/waiverdb-dev-integration-test.env index cc605f7..678e094 100644 --- a/openshift/pipelines/jobs/waiverdb-dev-integration-test.env +++ b/openshift/pipelines/jobs/waiverdb-dev-integration-test.env @@ -1,2 +1 @@ NAME=waiverdb-dev-integration-test -IMAGE=quay.io/factory2/waiverdb:latest diff --git a/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.env b/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.env index b4ec4a5..65b5e1b 100644 --- a/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.env +++ b/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-prod.env @@ -1,5 +1,4 @@ NAME=waiverdb-greenwave-promote-to-prod -SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb TARGET_TAG=prod DECISION_CONTEXT_REGEX=c3i_promote_stage_to_prod MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-prod.VirtualTopic.eng.greenwave.decision.update diff --git a/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.env b/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.env index 22fe0eb..17f4973 100644 --- a/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.env +++ b/openshift/pipelines/jobs/waiverdb-greenwave-promote-to-stage.env @@ -1,5 +1,4 @@ NAME=waiverdb-greenwave-promote-to-stage -SOURCE_CONTAINER_REPO=quay.io/factory2/waiverdb TARGET_TAG=stage DECISION_CONTEXT_REGEX=c3i_promote_dev_to_stage MESSAGING_TOPIC=Consumer.rh-jenkins-ci-plugin.c3i-waiverdb-promote-to-stage.VirtualTopic.eng.greenwave.decision.update diff --git a/openshift/pipelines/jobs/waiverdb-postmerge.env b/openshift/pipelines/jobs/waiverdb-postmerge.env index 320dbfe..c59617a 100644 --- a/openshift/pipelines/jobs/waiverdb-postmerge.env +++ b/openshift/pipelines/jobs/waiverdb-postmerge.env @@ -1,3 +1,2 @@ NAME=waiverdb-postmerge PAGURE_DOC_REPO_NAME= # Temporarily disable doc push to workaround https://pagure.io/pagure/issue/3919. Remove this line when it is fixed. -MAIL_ADDRESS=pnt-factory2-devel@redhat.com diff --git a/openshift/pipelines/jobs/waiverdb-prod-integration-test.env b/openshift/pipelines/jobs/waiverdb-prod-integration-test.env index 435af76..2f7cd4f 100644 --- a/openshift/pipelines/jobs/waiverdb-prod-integration-test.env +++ b/openshift/pipelines/jobs/waiverdb-prod-integration-test.env @@ -2,4 +2,3 @@ NAME=waiverdb-prod-integration-test IMAGE=quay.io/factory2/waiverdb:prod ENVIRONMENT=prod BACKEND_INTEGRATION_TEST_JOB=factory2-prod-integration-test -BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i diff --git a/openshift/pipelines/jobs/waiverdb-stage-integration-test.env b/openshift/pipelines/jobs/waiverdb-stage-integration-test.env index 907bd88..2e9ad43 100644 --- a/openshift/pipelines/jobs/waiverdb-stage-integration-test.env +++ b/openshift/pipelines/jobs/waiverdb-stage-integration-test.env @@ -2,4 +2,3 @@ NAME=waiverdb-stage-integration-test IMAGE=quay.io/factory2/waiverdb:stage ENVIRONMENT=stage BACKEND_INTEGRATION_TEST_JOB=factory2-stage-integration-test -BACKEND_INTEGRATION_TEST_JOB_NAMESPACE=c3i diff --git a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml index 137ce28..660ba4e 100644 --- a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml +++ b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml @@ -199,7 +199,8 @@ objects: script { dir('openshift/pipelines') { sh ''' - make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" + # service account cannot update rolebinding + make install JOBS_DIR="${PIPELINE_UPDATE_JOBS_DIR}" FAIL_ON_ERROR=false ''' } } From 35b7c4a34543aacfb809dcb7fe54a722568cf5d2 Mon Sep 17 00:00:00 2001 From: Michal Kovarik Date: Sep 27 2019 06:54:05 +0000 Subject: [PATCH 2/3] Set HOME for Jenkins slaves HOME environment variable is to /home/jenkins which is not writable. Jenkins slave container has default value of HOME set to /var/lib/jenkins. HOME variable is probably changed by jenkins plugin. --- diff --git a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile index d47fcc1..ca5beda 100644 --- a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile @@ -35,6 +35,8 @@ pipeline { value: '/tmp/passwd' - name: NSS_WRAPPER_GROUP value: '/etc/group' + - name: HOME + value: '/var/lib/jenkins' volumeMounts: - name: postgresql-socket mountPath: /var/run/postgresql diff --git a/openshift/pipelines/templates/waiverdb-full-integration-test.Jenkinsfile b/openshift/pipelines/templates/waiverdb-full-integration-test.Jenkinsfile index 3bd685d..4b8b923 100644 --- a/openshift/pipelines/templates/waiverdb-full-integration-test.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-full-integration-test.Jenkinsfile @@ -18,6 +18,9 @@ pipeline { - name: jnlp image: "${params.JENKINS_AGENT_IMAGE}" imagePullPolicy: Always + env: + - name: HOME + value: "/var/lib/jenkins" resources: requests: memory: 512Mi diff --git a/openshift/pipelines/templates/waiverdb-greenwave-trigger.Jenkinsfile b/openshift/pipelines/templates/waiverdb-greenwave-trigger.Jenkinsfile index 40a1e81..5d0db45 100644 --- a/openshift/pipelines/templates/waiverdb-greenwave-trigger.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-greenwave-trigger.Jenkinsfile @@ -43,6 +43,9 @@ podTemplate( image: ${params.JENKINS_AGENT_IMAGE} imagePullPolicy: Always tty: true + env: + - name: HOME + value: "/var/lib/jenkins" resources: requests: memory: 256Mi diff --git a/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile b/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile index c76272d..c33f3d9 100644 --- a/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-image-promotion.Jenkinsfile @@ -20,6 +20,8 @@ pipeline { imagePullPolicy: Always tty: true env: + - name: HOME + value: "/var/lib/jenkins" - name: REGISTRY_CREDENTIALS valueFrom: secretKeyRef: diff --git a/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile index f7316ed..4a93c69 100644 --- a/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-integration-test.Jenkinsfile @@ -26,6 +26,8 @@ pipeline { secretKeyRef: name: "${params.CONTAINER_REGISTRY_CREDENTIALS}" key: '.dockerconfigjson' + - name: HOME + value: "/var/lib/jenkins" resources: requests: memory: 512Mi diff --git a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml index 660ba4e..8130791 100644 --- a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml +++ b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml @@ -111,6 +111,9 @@ objects: image: "${JENKINS_AGENT_IMAGE}" imagePullPolicy: Always tty: true + env: + - name: HOME + value: "/var/lib/jenkins" resources: requests: memory: 378Mi diff --git a/openshift/pipelines/templates/waiverdb-repotracker-trigger.Jenkinsfile b/openshift/pipelines/templates/waiverdb-repotracker-trigger.Jenkinsfile index 7a4b8c5..3e793fa 100644 --- a/openshift/pipelines/templates/waiverdb-repotracker-trigger.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-repotracker-trigger.Jenkinsfile @@ -38,6 +38,9 @@ podTemplate( image: ${params.JENKINS_AGENT_IMAGE} imagePullPolicy: Always tty: true + env: + - name: HOME + value: "/var/lib/jenkins" resources: requests: memory: 256Mi From b88620d4f30acdbbbed9fa93c63f031d624c49c1 Mon Sep 17 00:00:00 2001 From: Michal Kovarik Date: Sep 27 2019 07:04:30 +0000 Subject: [PATCH 3/3] Do premerge testing with Jenkins files from the PR Varibales for git checkout (WAIVERDB_GIT_REPO, WAIVERDB_GIT_BRANCH) were evaluated during openshift template applications. Which caused that PRs were tested with Jenkins file in master branch intead of PR ones. Openshift BuildConfig is not able to specify git refs to be fetched and PR refs are not available, to avoid this issue WAIVERDB_GIT_REF_COMMIT contains SHA of last PR (or can be master branch). --- diff --git a/openshift/pipelines/templates/waiverdb-build-template.yaml b/openshift/pipelines/templates/waiverdb-build-template.yaml index 998ce68..3c9b3ad 100644 --- a/openshift/pipelines/templates/waiverdb-build-template.yaml +++ b/openshift/pipelines/templates/waiverdb-build-template.yaml @@ -41,6 +41,11 @@ parameters: description: Default WaiverDB Git repo ref in which to run dev tests against required: true value: master +- name: WAIVERDB_GIT_REF_COMMIT + displayName: WaiverDB Git repo ref reachable by Jenkins + description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file + required: true + value: master - name: WAIVERDB_MAIN_BRANCH displayName: Name of the main branch. description: If WAIVERDB_MAIN_BRANCH equals WAIVERDB_GIT_REF, this is a post-merge build, otherwise it's a pre-merge build. @@ -178,8 +183,9 @@ objects: completionDeadlineSeconds: 1800 source: git: - uri: "${WAIVERDB_GIT_REPO}" - ref: "${WAIVERDB_MAIN_BRANCH}" + # Using jenkins variable, not template + uri: "$WAIVERDB_GIT_REPO" + ref: "$WAIVERDB_GIT_REF_COMMIT" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -188,6 +194,8 @@ objects: value: "${WAIVERDB_GIT_REPO}" - name: "WAIVERDB_GIT_REF" value: "${WAIVERDB_GIT_REF}" + - name: "WAIVERDB_GIT_REF_COMMIT" + value: "${WAIVERDB_GIT_REF_COMMIT}" - name: "JENKINS_AGENT_CLOUD_NAME" value: "${JENKINS_AGENT_CLOUD_NAME}" - name: "JENKINS_AGENT_IMAGE" diff --git a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile index ca5beda..641c035 100644 --- a/openshift/pipelines/templates/waiverdb-build.Jenkinsfile +++ b/openshift/pipelines/templates/waiverdb-build.Jenkinsfile @@ -336,6 +336,7 @@ pipeline { '-e', "WAIVERDB_GIT_REPO=${params.WAIVERDB_GIT_REPO}", '-e', "IMAGE=${env.RESULTING_IMAGE_REPO}:${env.RESULTING_TAG}", '-e', "WAIVERDB_GIT_REF=${env.PR_NO ? env.WAIVERDB_GIT_REF : env.WAIVERDB_GIT_COMMIT}", + '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}", '-e', "IMAGE_IS_SCRATCH=${params.WAIVERDB_GIT_REF != params.WAIVERDB_MAIN_BRANCH}", ) c3i.wait(buildSelector.name()) diff --git a/openshift/pipelines/templates/waiverdb-integration-test-template.yaml b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml index 17021ee..e23517a 100644 --- a/openshift/pipelines/templates/waiverdb-integration-test-template.yaml +++ b/openshift/pipelines/templates/waiverdb-integration-test-template.yaml @@ -27,6 +27,11 @@ parameters: description: Default WaiverDB Git repo ref in which to run functional tests against required: true value: master +- name: WAIVERDB_GIT_REF_COMMIT + displayName: WaiverDB Git repo ref reachable by Jenkins + description: Default WaiverDB Git repo ref in which to get Jenkins pipeline file + required: true + value: master - name: JENKINS_AGENT_IMAGE displayName: Container image for Jenkins slave pods required: true @@ -75,8 +80,9 @@ objects: completionDeadlineSeconds: 1800 source: git: - uri: "${WAIVERDB_GIT_REPO}" - ref: "${WAIVERDB_GIT_REF}" + # Using jenkins variable, not template + uri: "$WAIVERDB_GIT_REPO" + ref: "$WAIVERDB_GIT_REF_COMMIT" strategy: type: JenkinsPipeline jenkinsPipelineStrategy: @@ -85,6 +91,8 @@ objects: value: "${WAIVERDB_GIT_REPO}" - name: "WAIVERDB_GIT_REF" value: "${WAIVERDB_GIT_REF}" + - name: "WAIVERDB_GIT_REF_COMMIT" + value: "${WAIVERDB_GIT_REF_COMMIT}" - name: "IMAGE" value: "${IMAGE}" - name: IMAGE_IS_SCRATCH diff --git a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml index 8130791..46aa87d 100644 --- a/openshift/pipelines/templates/waiverdb-polling-pagure.yaml +++ b/openshift/pipelines/templates/waiverdb-polling-pagure.yaml @@ -218,6 +218,7 @@ objects: echo 'Starting a WaiverDB build run...' def devBuild = bcSelector.startBuild( '-e', "WAIVERDB_GIT_REF=${env.WAIVERDB_GIT_BRANCH}", + '-e', "WAIVERDB_GIT_REF_COMMIT=${env.WAIVERDB_GIT_COMMIT}", ) devBuild.watch { return !(it.object().status.phase in ["New", "Pending"])