From c95e281431a9c7157093e9410c20cf83a60e64c6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 1/11] Move the pagure_ci doc into the usage section --- diff --git a/doc/pagure_ci.rst b/doc/pagure_ci.rst deleted file mode 100644 index 6f76c31..0000000 --- a/doc/pagure_ci.rst +++ /dev/null @@ -1,135 +0,0 @@ -========= -Pagure CI -========= - -Pagure CI is a continuous integration tool using which the PR on the projects -can be tested and flaged with the status of the build. - -How to enable Pagure CI -======================= - -* Enable the Fedmsg plugin in pagure project setting . This will emit the message - to for consumer to consume it. - -* Fill in the Pagure CI form with the required details. - -:: - - Pagure Project Name - Jenkins Project Name - Jenkins Token - Jenkins Url - - All of which are required field. - -* The jenkins token is any string that you give here. The only thing that should - be kept in mind that this token should be same through out. - -* This will give a POST URL which will be used for Job Notification in Jenkins - -* The POST url will only appear only after you successfully submitted the form. - - -Configuring Jenkins -=================== - -Jenkins configuration is the most important part of how the Pagure CI works, -after you login to your Jenkins Instance. - -* Go to Manage Jenkins -> Configuire Global Security and under that select - `Project-based Matrix Authorization Strategy` - -* Add your username here and make sure to give that username all the permissions. - You should give all the permissions possible so that you save your self from - getting locked in Jenkins. - -* Download the following plugins: - -:: - - Build Authorization Root Plugin - Git Plugins - Notification Plugin - - -* Click on the New Item - -* Select Freestyle Project - -* Click OK and enter the name of the project, make sure the project name - you filled in the Pagure CI form should match the name you entered here. - -* Under 'Job Notification' click 'Add Endpoint' - -* Fields in Endpoint will be : - -:: - - FORMAT: JSON - PROTOCOL: HTTP - EVENT: Job Finalized - URL: - TIMEOUT: 3000 - LOG: 1 - -* Tick the build is parameterized - -* From the Add Parameter drop down select String Parameter - -* Two string parameters need to be created REPO and BRANCH - -* Source Code Management select Git and give the URL of the pagure project - -* Under Build Trigger click on Trigger build remotely and give the same token - that you gave in the Pagure CI form. - -* Under Build -> Add build step -> Execute Shell - -* In the box given enter the shell steps you want for testing your project. - - -Example Script - -:: - - if [ -n "$REPO" -a -n "$BRANCH" ]; then - git remote rm proposed || true - git remote add proposed "$REPO" - git fetch proposed - git checkout origin/master - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - git merge --no-ff "proposed/$BRANCH" -m "Merge PR" - fi - -How to install Pagure CI -======================== - -Pagure CI requires `fedmsg` to run since it uses a consumer to get messages -and take appropriate actions. The dependency that is required is `fedmdg-hubs`. -For that the steps are given. - -To install the dependencies required: - - `dnf install fedmsg-hub` - -`fedmsg` apart from the consumer require a file that tells to which cosumer -it should listen to. This file basically enable the consumer in PagureCI/. -For doing that, we need to place this file in appropriate directory. - - `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` - -Since the deployment is done using rpm, the next step is covered using `setup.py` -which binds the consumer with the environment, this is done while building the rpm -so if rpm is already built this is not explicitly required. - - `python setup.py install` - -Run the service: - - `sudo systemctl enable fedmsg-hub.service` - - `sudo systemctl start fedmsg-hub.service` - - - diff --git a/doc/usage/pagure_ci.rst b/doc/usage/pagure_ci.rst new file mode 100644 index 0000000..3d7780d --- /dev/null +++ b/doc/usage/pagure_ci.rst @@ -0,0 +1,132 @@ +========= +Pagure CI +========= + +Pagure CI is a continuous integration tool using which the PR on the projects +can be tested and flaged with the status of the build. + +How to enable Pagure CI +======================= + +* Enable the Fedmsg plugin in pagure project setting . This will emit the message + to for consumer to consume it. + +* Fill in the Pagure CI form with the required details. + +:: + + Pagure Project Name + Jenkins Project Name + Jenkins Token + Jenkins Url + + All of which are required field. + +* The jenkins token is any string that you give here. The only thing that should + be kept in mind that this token should be same through out. + +* This will give a POST URL which will be used for Job Notification in Jenkins + +* The POST url will only appear only after you successfully submitted the form. + + +Configuring Jenkins +=================== + +Jenkins configuration is the most important part of how the Pagure CI works, +after you login to your Jenkins Instance. + +* Go to Manage Jenkins -> Configuire Global Security and under that select + `Project-based Matrix Authorization Strategy` + +* Add your username here and make sure to give that username all the permissions. + You should give all the permissions possible so that you save your self from + getting locked in Jenkins. + +* Download the following plugins: + +:: + + Build Authorization Root Plugin + Git Plugins + Notification Plugin + + +* Click on the New Item + +* Select Freestyle Project + +* Click OK and enter the name of the project, make sure the project name + you filled in the Pagure CI form should match the name you entered here. + +* Under 'Job Notification' click 'Add Endpoint' + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: Job Finalized + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the build is parameterized + +* From the Add Parameter drop down select String Parameter + +* Two string parameters need to be created REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and give the same token + that you gave in the Pagure CI form. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + +How to install Pagure CI +======================== + +Pagure CI requires `fedmsg` to run since it uses a consumer to get messages +and take appropriate actions. The dependency that is required is `fedmdg-hubs`. +For that the steps are given. + +To install the dependencies required: + + `dnf install fedmsg-hub` + +`fedmsg` apart from the consumer require a file that tells to which cosumer +it should listen to. This file basically enable the consumer in PagureCI/. +For doing that, we need to place this file in appropriate directory. + + `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` + +Since the deployment is done using rpm, the next step is covered using `setup.py` +which binds the consumer with the environment, this is done while building the rpm +so if rpm is already built this is not explicitly required. + + `python setup.py install` + +Run the service: + + `sudo systemctl enable fedmsg-hub.service` + + `sudo systemctl start fedmsg-hub.service` From 29fc18a1fdbc9f7c33a2189387c2154a9f028501 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 2/11] Make the pagure_ci home page something generic --- diff --git a/doc/usage/pagure_ci.rst b/doc/usage/pagure_ci.rst index 3d7780d..96004ff 100644 --- a/doc/usage/pagure_ci.rst +++ b/doc/usage/pagure_ci.rst @@ -1,132 +1,19 @@ -========= Pagure CI ========= -Pagure CI is a continuous integration tool using which the PR on the projects -can be tested and flaged with the status of the build. - -How to enable Pagure CI -======================= - -* Enable the Fedmsg plugin in pagure project setting . This will emit the message - to for consumer to consume it. - -* Fill in the Pagure CI form with the required details. - -:: - - Pagure Project Name - Jenkins Project Name - Jenkins Token - Jenkins Url - - All of which are required field. - -* The jenkins token is any string that you give here. The only thing that should - be kept in mind that this token should be same through out. - -* This will give a POST URL which will be used for Job Notification in Jenkins - -* The POST url will only appear only after you successfully submitted the form. - - -Configuring Jenkins -=================== - -Jenkins configuration is the most important part of how the Pagure CI works, -after you login to your Jenkins Instance. - -* Go to Manage Jenkins -> Configuire Global Security and under that select - `Project-based Matrix Authorization Strategy` - -* Add your username here and make sure to give that username all the permissions. - You should give all the permissions possible so that you save your self from - getting locked in Jenkins. - -* Download the following plugins: - -:: - - Build Authorization Root Plugin - Git Plugins - Notification Plugin - - -* Click on the New Item - -* Select Freestyle Project - -* Click OK and enter the name of the project, make sure the project name - you filled in the Pagure CI form should match the name you entered here. - -* Under 'Job Notification' click 'Add Endpoint' - -* Fields in Endpoint will be : - -:: - - FORMAT: JSON - PROTOCOL: HTTP - EVENT: Job Finalized - URL: - TIMEOUT: 3000 - LOG: 1 - -* Tick the build is parameterized - -* From the Add Parameter drop down select String Parameter - -* Two string parameters need to be created REPO and BRANCH - -* Source Code Management select Git and give the URL of the pagure project - -* Under Build Trigger click on Trigger build remotely and give the same token - that you gave in the Pagure CI form. - -* Under Build -> Add build step -> Execute Shell - -* In the box given enter the shell steps you want for testing your project. - - -Example Script - -:: - - if [ -n "$REPO" -a -n "$BRANCH" ]; then - git remote rm proposed || true - git remote add proposed "$REPO" - git fetch proposed - git checkout origin/master - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - git merge --no-ff "proposed/$BRANCH" -m "Merge PR" - fi - -How to install Pagure CI -======================== - -Pagure CI requires `fedmsg` to run since it uses a consumer to get messages -and take appropriate actions. The dependency that is required is `fedmdg-hubs`. -For that the steps are given. - -To install the dependencies required: - - `dnf install fedmsg-hub` - -`fedmsg` apart from the consumer require a file that tells to which cosumer -it should listen to. This file basically enable the consumer in PagureCI/. -For doing that, we need to place this file in appropriate directory. +Pagure CI is a service integrating the results of Continuous Integration (CI) +services, such as jenkins or travis-ci, into pull-requests opened against +your project on pagure. - `sudo cp pagure/fedmsg.d/pagure_ci.py /etc/fedmsg.d/` -Since the deployment is done using rpm, the next step is covered using `setup.py` -which binds the consumer with the environment, this is done while building the rpm -so if rpm is already built this is not explicitly required. +.. note: By default pagure-ci is off, an admin of your pagure instance will + need to configure it to support one or more CI services. Check the + configuration section on how to do that. - `python setup.py install` -Run the service: +Contents: - `sudo systemctl enable fedmsg-hub.service` +.. toctree:: + :maxdepth: 2 - `sudo systemctl start fedmsg-hub.service` + usage/pagure_ci_jenkins From 5854255dd4e3ca6d387b695591dcd9f4552b76fb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 3/11] Introduce the pagure_ci_jenkins doc, specific for jenkings/pagure integration --- diff --git a/doc/usage/pagure_ci_jenkins.rst b/doc/usage/pagure_ci_jenkins.rst new file mode 100644 index 0000000..ec54ebf --- /dev/null +++ b/doc/usage/pagure_ci_jenkins.rst @@ -0,0 +1,92 @@ +Jenkins with Pagure-ci +====================== + +Jenkins is a Continuous Integration service that can be configured to be +integrated with pagure. + +This document describe the steps needed to make it work. + + +How to enable Pagure CI +======================= + +* Visit the settings page of your project + +* Scroll down to the `Hooks` section and click on `Pagure CI` + +* Select the type of CI service you want + +* Enter the URL to the project on the CI service. For example, if your + project is running at `http://jenkins.fedoraproject.org` you will need to + enter the url: `http://jenkins.fedoraproject.org/job/` + +* Tick the checkbox activating the hook. + + +These steps will activate the hook, after reloading the page or the tab, you +will be given access to two important values: the token used to trigger the +build on jenkins and the URL used by jenkins to report the status of the +build. +Keep these two available when configuring jenkins for your project. + + +Configure Jenkins +================= + +These steps can only be made by the admins of your jenkins instance, but +they only need to be made once. + +* Download the following plugins: + + * `Git Plugin `_ + * `Notification Plugin `_ + + +Configure your project on Jenkins +================================= + +* Go to the `Configure` page of your project + +* Under `Job Notification` click `Add Endpoint` + +* Fields in Endpoint will be : + +:: + + FORMAT: JSON + PROTOCOL: HTTP + EVENT: Job Finalized + URL: + TIMEOUT: 3000 + LOG: 1 + +* Tick the checkbox `This build is parameterized` + +* Add two `String Parameters` named REPO and BRANCH + +* Source Code Management select Git and give the URL of the pagure project + +* Under Build Trigger click on Trigger build remotely and specify the token + given by pagure. + +* Under Build -> Add build step -> Execute Shell + +* In the box given enter the shell steps you want for testing your project. + + +Example Script + +:: + + # Script specific for Pull-Request build + if [ -n "$REPO" -a -n "$BRANCH" ]; then + git remote rm proposed || true + git remote add proposed "$REPO" + git fetch proposed + git checkout origin/master + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + git merge --no-ff "proposed/$BRANCH" -m "Merge PR" + fi + + # Part of the script specific to how you run the tests on your project From c1b01d554ac784da47859bf373a720b8877c6d38 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 4/11] Include pagure_ci in the usage doc --- diff --git a/doc/usage.rst b/doc/usage.rst index aee0095..3996a8a 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -41,3 +41,4 @@ Contents: usage/pr_custom_page usage/theming usage/upgrade_db + usage/pagure_ci From 219d3119c02789ee7592a1bafe83e66fb1e05ad4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 5/11] Do not repeat information from the doc, rather point to it --- diff --git a/pagure-ci/README.rst b/pagure-ci/README.rst index 0ab6362..9dac42e 100644 --- a/pagure-ci/README.rst +++ b/pagure-ci/README.rst @@ -9,76 +9,5 @@ dependencies are resolved. PAGURE_CONFIG=/path/to/config PYTHONPATH=. python pagure-ci/pagure_ci_server.py - -Configure Jenkins -================= - -Jenkins configuration is the most important part of how the Pagure CI works, -after you login to your Jenkins Instance. - - -* Go to Manage Jenkins -> Configuire Global Security and under that select - 'Project-based Matrix Authorization Strategy' - -* Add a user and give all the permission to that user. - -* Download the following plugins: - - * Build Authorization Root Plugin - * `Git Plugin `_ - * `Notification Plugin `_ - - -Configure your project on Jenkins -================================= - -* Start by enabling the `Pagure CI` hook in the settings of your project on - pagure. This will provide you two values needed to configure your project - on jenkins: a token and an URL that jenkins calls to return the results - of a build. - -* Go to the `Configure` page of your project - -* Under `Job Notification` click `Add Endpoint` - -* Fields in Endpoint will be : - -:: - - FORMAT: JSON - PROTOCOL: HTTP - EVENT: Job Finalized - URL: - TIMEOUT: 3000 - LOG: 1 - -* Tick the checkbox `This build is parameterized` - -* Add two `String Parameters` named REPO and BRANCH - -* Source Code Management select Git and give the URL of the pagure project - -* Under Build Trigger click on Trigger build remotely and specify the token - given by pagure. - -* Under Build -> Add build step -> Execute Shell - -* In the box given enter the shell steps you want for testing your project. - - -Example Script - -:: - - # Script specific for Pull-Request build - if [ -n "$REPO" -a -n "$BRANCH" ]; then - git remote rm proposed || true - git remote add proposed "$REPO" - git fetch proposed - git checkout origin/master - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - git merge --no-ff "proposed/$BRANCH" -m "Merge PR" - fi - - # Part of the script specific to how you run the tests on your project +Check `doc/usage/pagure_ci.rst` for further information on how to set up +and configure your project on both pagure and the CI services From 5aef08047002e6a875dddf964447aab37f0e148b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 6/11] Fix including the pagure_ci_jenkins page --- diff --git a/doc/usage/pagure_ci.rst b/doc/usage/pagure_ci.rst index 96004ff..e0d337f 100644 --- a/doc/usage/pagure_ci.rst +++ b/doc/usage/pagure_ci.rst @@ -16,4 +16,4 @@ Contents: .. toctree:: :maxdepth: 2 - usage/pagure_ci_jenkins + pagure_ci_jenkins From 2323afda018f49bf595797888703b0c64d5e01f1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 7/11] Adjust the title levels in the pagure_ci_jenkins doc --- diff --git a/doc/usage/pagure_ci_jenkins.rst b/doc/usage/pagure_ci_jenkins.rst index ec54ebf..10a9a98 100644 --- a/doc/usage/pagure_ci_jenkins.rst +++ b/doc/usage/pagure_ci_jenkins.rst @@ -8,7 +8,7 @@ This document describe the steps needed to make it work. How to enable Pagure CI -======================= +----------------------- * Visit the settings page of your project @@ -31,7 +31,7 @@ Keep these two available when configuring jenkins for your project. Configure Jenkins -================= +----------------- These steps can only be made by the admins of your jenkins instance, but they only need to be made once. @@ -43,7 +43,7 @@ they only need to be made once. Configure your project on Jenkins -================================= +--------------------------------- * Go to the `Configure` page of your project From 1d815a409bcfd8d76b62f52274953c3ecadbb9a1 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 8/11] Document PAGURE_CI_SERVICES in the documentation --- diff --git a/doc/configuration.rst b/doc/configuration.rst index 42a36c0..ca83c75 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -515,3 +515,18 @@ which is useful for pagure instances running since before 1.3 but is not for newer instances. Defaults to: ``False``. + + +PAGURE_CI_SERVICES +~~~~~~~~~~~~~~~~~~ + +Pagure can be configure to integrate results of a Continuous Integration (CI) +service to pull-requests open against a project. + +To enable this integration, follow the documentation on how to install +pagure-ci and set this configuration key to ``['jenkins']`` (Jenkins being +the only CI service supported at the moment). + +Defaults to: ``None``. + +.. warning:: Requires `Redis` to be configured and running. From 499dad90eec78b29a4e3420485e9dcb64c796325 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:49 +0000 Subject: [PATCH 9/11] Add a documentation on how to install pagure-ci --- diff --git a/doc/index.rst b/doc/index.rst index 268c3e3..3daea4c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -34,6 +34,7 @@ Contents: install_milter install_evs install_webhooks + install_pagure_ci configuration development usage diff --git a/doc/install_pagure_ci.rst b/doc/install_pagure_ci.rst new file mode 100644 index 0000000..a1158e5 --- /dev/null +++ b/doc/install_pagure_ci.rst @@ -0,0 +1,61 @@ +Installing pagure-ci +==================== + +A CI stands for `Continuous Integration +`_. Pagure can be +configured to integrate results coming from CI services, such as `Jenkins +`_ on pull-request opened +against the project. + + +.. note: Currently, pagure only supports `Jenkins` but we welcome help to + integrate pagure with other services such as `travis-ci + `_. + + +Configure your system +--------------------- + +* Install the required dependencies + +:: + + python-jenkins + python-redis + python-trollius-redis + python-trollius + +.. note:: We ship a systemd unit file for pagure_ci but we welcome patches + for scripts for other init systems. + + +* Install the files of pagure-ci as follow: + ++--------------------------------------+---------------------------------------------------+ +| Source | Destination | ++======================================+===================================================+ +| ``pagure-ci/pagure_ci_server.py`` | ``/usr/libexec/pagure-ci/pagure_ci_server.py`` | ++--------------------------------------+---------------------------------------------------+ +| ``pagure-ci/pagure_ci.service`` | ``/etc/systemd/system/pagure_ci.service`` | ++--------------------------------------+---------------------------------------------------+ + +The first file is the pagure-ci service itself, triggering the build on the +CI service when there is a new pull-request or a change to an existing one. + +The second file is the systemd service file. + +* Configure your pagure instance to support CI, add the following to your + configuration file + +:: + + PAGURE_CI_SERVICES = ['jenkins'] + +* Activate the service and ensure it's started upon boot: + +:: + + systemctl enable redis + systemctl start redis + systemctl enable pagure_ci + systemctl start pagure_ci From 4e920083a654a11877df17cb461e2e1152ab0aed Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 08:44:50 +0000 Subject: [PATCH 10/11] Adjust syntax for a note --- diff --git a/doc/usage/pagure_ci.rst b/doc/usage/pagure_ci.rst index e0d337f..8176c5b 100644 --- a/doc/usage/pagure_ci.rst +++ b/doc/usage/pagure_ci.rst @@ -6,7 +6,7 @@ services, such as jenkins or travis-ci, into pull-requests opened against your project on pagure. -.. note: By default pagure-ci is off, an admin of your pagure instance will +.. note:: By default pagure-ci is off, an admin of your pagure instance will need to configure it to support one or more CI services. Check the configuration section on how to do that. From c68e929ed03ba760bc3606012d444ba85fe3eb36 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Aug 11 2016 09:14:51 +0000 Subject: [PATCH 11/11] Fix unit-tests We were retrieving the commit hash of the wrong repo then trying to look for it on the right page. This should fix that. --- diff --git a/tests/test_pagure_flask_ui_repo_slash_name.py b/tests/test_pagure_flask_ui_repo_slash_name.py index 4ecd2a3..349d7f8 100644 --- a/tests/test_pagure_flask_ui_repo_slash_name.py +++ b/tests/test_pagure_flask_ui_repo_slash_name.py @@ -239,11 +239,18 @@ class PagureFlaskSlashInNametests(tests.Modeltests): output.data) # Try accessing the commit - gitrepo = os.path.join(tests.HERE, 'repos', 'test.git') + gitrepo = os.path.join(tests.HERE, 'repos', 'forks/test.git') repo = pygit2.Repository(gitrepo) master_branch = repo.lookup_branch('master') first_commit = master_branch.get_object().hex + output = self.app.get('/forks/test/commits') + self.assertEqual(output.status_code, 200) + self.assertIn(first_commit, output.data) + self.assertIn( + 'Commit - forks/test ', output.data)