From f8b7a2ba4b3c71e32aaaa55ce9e2d1e0f62bf702 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Jan 21 2024 23:19:24 +0000 Subject: Drop repoSpanner support RepoSpanner was an interesting project to attempt to scale the Git backend for Pagure, but it was never completed and eventually abandoned. Signed-off-by: Neal Gompa --- diff --git a/.gitignore b/.gitignore index 12d9b09..576ddff 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,5 @@ dev/containers/test_env Pipfile # A possible symlink for the testsuite -/repospanner /repobridge /repohookrunner diff --git a/UPGRADING.rst b/UPGRADING.rst index 07d5962..7ca0682 100644 --- a/UPGRADING.rst +++ b/UPGRADING.rst @@ -1,6 +1,29 @@ Upgrading Pagure ================ +From 5.13 to 6.0 +----------------- + +The 6.0 release contains database schema updates, so: + +* Update the data schema using alembic: ``alembic upgrade head`` + +(As usual, do your backups before). + +Removed features (in 6.0): + +* repoSpanner, which was never completed and eventually abandoned. + +Configuration keys removed (in 6.0); + +* REPOSPANNER_PSEUDO_FOLDER +* REPOSPANNER_NEW_REPO +* REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE +* REPOSPANNER_NEW_FORK +* REPOSPANNER_ADMIN_MIGRATION +* REPOSPANNER_REGIONS + + From 5.12 to 5.13 ----------------- diff --git a/alembic/versions/5df8314dfc13_drop_repospanner_region_column.py b/alembic/versions/5df8314dfc13_drop_repospanner_region_column.py new file mode 100644 index 0000000..8a8315b --- /dev/null +++ b/alembic/versions/5df8314dfc13_drop_repospanner_region_column.py @@ -0,0 +1,23 @@ +"""Drop repoSpanner region column + +Revision ID: 5df8314dfc13 +Revises: c0bffa4e8fbc +Create Date: 2024-01-21 23:07:25.610148 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5df8314dfc13' +down_revision = 'c0bffa4e8fbc' + + +def upgrade(): + op.drop_column('projects', 'repospanner_region') + + +def downgrade(): + op.add_column('projects', sa.Column('repospanner_region', sa.Text(), nullable=True)) diff --git a/dev/ansible/roles/pagure-dev/files/pagure.cfg b/dev/ansible/roles/pagure-dev/files/pagure.cfg index c839a93..7085dea 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure.cfg +++ b/dev/ansible/roles/pagure-dev/files/pagure.cfg @@ -63,7 +63,7 @@ SSH_KEYS_OPTIONS = ( 'restrict,command="/usr/bin/python3 /srv/pagure/files/aclchecker.py %(username)s"' ) -SSH_COMMAND_NON_REPOSPANNER = ([ +SSH_COMMAND = ([ "/usr/bin/%(cmd)s", "/srv/git/repositories/%(reponame)s", ], {"GL_USER": "%(username)s"}) diff --git a/doc/configuration.rst b/doc/configuration.rst index 8ca2e63..9053c5c 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -294,7 +294,7 @@ To use this variant, set the following in ``pagure.cfg``: SSH_KEYS_USERNAME_EXPECT = "git" - SSH_COMMAND_NON_REPOSPANNER = ([ + SSH_COMMAND = ([ "/usr/bin/%(cmd)s", "/srv/git/repositories/%(reponame)s", ], {"GL_USER": "%(username)s"}) @@ -329,7 +329,7 @@ set the following to ``pagure.cfg``: HTTP_REPO_ACCESS_GITOLITE = None - SSH_COMMAND_NON_REPOSPANNER = ([ + SSH_COMMAND = ([ "/usr/bin/%(cmd)s", "/srv/git/repositories/%(reponame)s", ], {"GL_USER": "%(username)s"}) @@ -353,7 +353,7 @@ Using Gitolite also requires setting the following in ``pagure.cfg``: HTTP_REPO_ACCESS_GITOLITE = "/usr/share/gitolite3/gitolite-shell" - SSH_COMMAND_NON_REPOSPANNER = ( + SSH_COMMAND = ( [ "/usr/share/gitolite3/gitolite-shell", "%(username)s", @@ -1709,9 +1709,6 @@ This is especially useful if repositories are stored on NFS (or similar network storage), where file metadata access is expensive - having unpacked objects in repositories requires *a lot* of metadata reads. -Note that the garbage collection is only run on repos that are not on -repoSpanner. - Defaults to: ``False`` @@ -1965,95 +1962,6 @@ Defaults to: ]) -RepoSpanner Options -------------------- - -Pagure can be integrated with `repoSpanner `_ -allowing to deploy pagure in a load-balanced environment since the git -repositories are then synced across multiple servers simultaneously. - -Support for this integration has been included in Pagure version 5.0 and higher. - -Here below are the different options one can/should use to integrate pagure -with repoSpanner. - -REPOBRIDGE_BINARY -~~~~~~~~~~~~~~~~~ - -This should contain the path to the repoBridge binary, which is used for pushing -and pulling to/from repoSpanner. - -Defaults to: ``/usr/libexec/repobridge``. - - -REPOSPANNER_NEW_REPO -~~~~~~~~~~~~~~~~~~~~ - -This configuration key instructs pagure to create new git repositories on -repoSpanner or not. -Its value should be the region in which the new git repositories should be -created on. - -Defaults to: ``None``. - -REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This configuration key can be used to let pagure admin override the default -region used when creating new git repositories on repoSpanner. -Its value should be a boolean. - -Defaults to: ``False`` - -REPOSPANNER_NEW_FORK -~~~~~~~~~~~~~~~~~~~~ - -This configuration key instructs pagure on where/how to create new git -repositories for the forks with repoSpanner. -If ``None``, git repositories for forks are created outside of repoSpanner -entirely. -If ``True``, git repositories for forks are created in the same region as -the parent project. -Otherwise, a region can be directly specified where git repositories for -forks will be created. - -Defaults to: ``True`` - -REPOSPANNER_ADMIN_MIGRATION -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This configuration key can be used to let admin manually migrate individual -project into repoSpanner once it is set up. - -Defaults to: ``False`` - -REPOSPANNER_REGIONS -~~~~~~~~~~~~~~~~~~~ - -This configuration key can be used to specify the different region where repoSpanner -is deployed and thus with which this pagure instance can be integrated. - -An example entry could look like: - -:: - - REPOSPANNER_REGIONS = { - 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', - 'repo_prefix': 'pagure/', - 'hook': None, - 'ca': '/etc/pki/repospanner/pki/ca.crt', - 'admin_cert': {'cert': '/etc/pki/repospanner/pki/admin.crt', - 'key': '/etc/pki/repospanner/pki/admin.key'}, - 'push_cert': {'cert': '/etc/pki/repospanner/pki/pagure.crt', - 'key': '/etc/pki/repospanner/pki/pagure.key'}} - } - -If this configuration key is not defined, pagure will consider that it is -not set to be integrated with repoSpanner. - -Defaults to: ``{}`` - - SSH_KEYS_USERNAME_LOOKUP ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2098,16 +2006,10 @@ it's not possible to use the ``IP_ALLOWED_INTERNAL`` address list. Defaults to: ``None`` -SSH_COMMAND_REPOSPANNER -~~~~~~~~~~~~~~~~~~~~~~~ - -The command to run if a repository is on repospanner when aclchecker is in use. - - -SSH_COMMAND_NON_REPOSPANNER -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SSH_COMMAND +~~~~~~~~~~~ -The command to run if a repository is not on repospanner when aclchecker is in use. +The command to run when aclchecker is in use. diff --git a/files/aclchecker.py b/files/aclchecker.py index 9592845..5cf1591 100644 --- a/files/aclchecker.py +++ b/files/aclchecker.py @@ -90,10 +90,7 @@ if not result["access"]: # Now go run the configured command # We verified that cmd is either "git-receive-pack" or "git-send-pack" # and "path" is a path that points to a valid Pagure repository. -if result["region"]: - runner, env = pagure_config["SSH_COMMAND_REPOSPANNER"] -else: - runner, env = pagure_config["SSH_COMMAND_NON_REPOSPANNER"] +runner, env = pagure_config["SSH_COMMAND"] result.update({"username": remoteuser, "cmd": cmd}) diff --git a/files/pagure.cfg.sample b/files/pagure.cfg.sample index 5a6e1db..f7c654e 100644 --- a/files/pagure.cfg.sample +++ b/files/pagure.cfg.sample @@ -88,12 +88,6 @@ GIT_FOLDER = os.path.join( 'repos' ) -REPOSPANNER_PSEUDO_FOLDER = os.path.join( - os.path.abspath(os.path.dirname(__file__)), - '..', - 'pseudo' -) - ### Folder containing the clones for the remote pull-requests REMOTE_GIT_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), @@ -133,7 +127,7 @@ GL_BINDIR = None # SSH Information ### SSH commend for interacting with repositories -SSH_COMMAND_NON_REPOSPANNER = ( +SSH_COMMAND = ( [ "/usr/bin/%(cmd)s", os.path.join(GIT_FOLDER, "%(reponame)s"), @@ -234,30 +228,6 @@ APPLICATION_ROOT = '/' # compatibility in your URLs. OLD_VIEW_COMMIT_ENABLED = False -# repoSpanner integration settings -# https://repospanner.org/ -# Whether to create new repositories on repoSpanner by default. -# Either None or a region name. -REPOSPANNER_NEW_REPO = None -# Whether to allow admins to override region selection on creation. -REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE = False -# Whether to create new forks on repoSpanner. -# Either None (no repoSpanner), True (same as origin project) or a region name. -REPOSPANNER_NEW_FORK = True -# Whether to allow an admin to manually migrate an individual project. -REPOSPANNER_ADMIN_MIGRATION = False -# The repoSpanner regions to be used in this Pagure instance. -# Example entry: -# 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', -# 'repo_prefix': 'pagure/', -# 'hook': None, -# 'ca': '', -# 'admin_cert': {'cert': '', -# 'key': ''}, -# 'push_cert': {'cert': '', -# 'key': ''}} -REPOSPANNER_REGIONS = {} - # Path to the plugins configuration file that is used to load plugins. Please # look at files/plugins.cfg.sample for a configuration example. # PAGURE_PLUGINS_CONFIG = "/etc/pagure/plugins.cfg" diff --git a/files/pagure.spec b/files/pagure.spec index cef000c..e6a9eac 100644 --- a/files/pagure.spec +++ b/files/pagure.spec @@ -332,8 +332,7 @@ sed -e "s|#!/usr/bin/env python|#!%{__python}|" -i \ $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/*.py \ $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/hookrunner \ $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/post-receive \ - $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/pre-receive \ - $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/repospannerhook + $RPM_BUILD_ROOT/%{python_sitelib}/pagure/hooks/files/pre-receive # Switch interpreter for systemd units sed -e "s|/usr/bin/python|%{__python}|g" -i $RPM_BUILD_ROOT/%{_unitdir}/*.service diff --git a/pagure/api/project.py b/pagure/api/project.py index 33d0b6f..52bf413 100644 --- a/pagure/api/project.py +++ b/pagure/api/project.py @@ -1411,12 +1411,6 @@ def api_new_project(): | | | | already a coresopnding | | | | | git repository on disk | +----------------------------+---------+--------------+---------------------------+ - | ``repospanner_region`` | boolean | Optional | | Only available to admins| - | | | | this option allows them | - | | | | to override the default | - | | | | respoSpanner region | - | | | | configured | - +----------------------------+---------+--------------+---------------------------+ | ``wait`` | boolean | Optional | | A boolean to specify if | | | | | this API call should | | | | | return a taskid or if it| @@ -1480,10 +1474,6 @@ def api_new_project(): private = False if pagure_config.get("PRIVATE_PROJECTS", False): private = form.private.data - if form.repospanner_region: - repospanner_region = form.repospanner_region.data - else: - repospanner_region = None if form.ignore_existing_repos: ignore_existing_repos = form.ignore_existing_repos.data else: @@ -1500,7 +1490,6 @@ def api_new_project(): flask.g.session, name=name, namespace=namespace, - repospanner_region=repospanner_region, ignore_existing_repo=ignore_existing_repos, description=description, private=private, diff --git a/pagure/cli/admin.py b/pagure/cli/admin.py index 80ca872..70d862c 100644 --- a/pagure/cli/admin.py +++ b/pagure/cli/admin.py @@ -15,11 +15,9 @@ import datetime import logging import os import sys -from string import Template import arrow import pygit2 -import requests from six.moves import input if "PAGURE_CONFIG" not in os.environ and os.path.exists( @@ -382,21 +380,6 @@ def _parser_block_user(subparser): local_parser.set_defaults(func=do_block_user) -def _parser_upload_repospanner_hooks(subparser): - """Set up the CLI argument parser to upload repospanner hook. - - Args: - subparser: An argparse subparser - """ - local_parser = subparser.add_parser( - "upload-repospanner-hook", help="Upload repoSpanner hook script" - ) - local_parser.add_argument( - "region", help="repoSpanner region where to " "upload hook" - ) - local_parser.set_defaults(func=do_upload_repospanner_hooks) - - def _parser_ensure_project_hooks(subparser): """Set up the CLI argument parser to ensure project hooks are setup @@ -407,9 +390,6 @@ def _parser_ensure_project_hooks(subparser): "ensure-project-hooks", help="Ensure all projects have their hooks setup", ) - local_parser.add_argument( - "hook", help="repoSpanner hook ID to set", default=None - ) local_parser.set_defaults(func=do_ensure_project_hooks) @@ -593,9 +573,6 @@ def parse_arguments(args=None): # block-user _parser_block_user(subparser) - # upload-repospanner-hooks - _parser_upload_repospanner_hooks(subparser) - # ensure-project-hooks _parser_ensure_project_hooks(subparser) @@ -1378,52 +1355,6 @@ def do_block_user(args): session.commit() -def do_upload_repospanner_hooks(args): - """Upload hooks to repoSpanner - - Args: - args (argparse.Namespace): Parsed arguments - """ - regioninfo = pagure.config.config["REPOSPANNER_REGIONS"].get(args.region) - if not regioninfo: - raise ValueError( - "repoSpanner region %s not in config file" % args.region - ) - - env = { - "config": os.environ.get("PAGURE_CONFIG", "/etc/pagure/pagure.cfg"), - "pypath": os.environ.get("PYTHONPATH", "None"), - } - sourcefile = os.path.abspath( - os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "../hooks/files/repospannerhook", - ) - ) - with open(sourcefile, "r") as source: - template = source.read() - hookcontents = Template(template).substitute(env) - - resp = requests.post( - "%s/admin/hook/admin.git/upload" % regioninfo["url"], - data=hookcontents, - headers={"X-Object-Size": str(len(hookcontents))}, - verify=regioninfo["ca"], - cert=( - regioninfo["admin_cert"]["cert"], - regioninfo["admin_cert"]["key"], - ), - ) - resp.raise_for_status() - resp = resp.json() - _log.debug("Response json: %s", resp) - if not resp["Success"]: - raise Exception("Error in repoSpanner API call: %s" % resp["Error"]) - hook = resp["Info"] - print("Hook ID for region %s: %s" % (args.region, hook)) - return hook - - def do_ensure_project_hooks(args): """Ensures that all projects have their hooks setup @@ -1437,9 +1368,7 @@ def do_ensure_project_hooks(args): for project in query.all(): print("Ensuring hooks for %s" % project.fullname) projects.append(project.fullname) - pagure.lib.git.set_up_project_hooks( - project, project.repospanner_region, hook=args.hook - ) + pagure.lib.git.set_up_project_hooks(project, hook=args.hook) return projects diff --git a/pagure/default_config.py b/pagure/default_config.py index 5a743a1..b3e3996 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -173,11 +173,6 @@ ATTACHMENTS_FOLDER = os.path.join( os.path.abspath(os.path.dirname(__file__)), "..", "lcl", "attachments" ) -# Folder for repoSpanner pseudo repos -REPOSPANNER_PSEUDO_FOLDER = os.path.join( - os.path.abspath(os.path.dirname(__file__)), "..", "lcl", "pseudo" -) - # Whether to enable scanning for viruses in attachments VIRUS_SCAN_ATTACHMENTS = False @@ -222,8 +217,7 @@ GL_BINDIR = None # Whether or not to run "git gc --auto" after every change to a project -# This will only run for projects not on repospanner and will use -# default git config values +# This will use default git config values # See https://git-scm.com/docs/git-gc#git-gc---auto for more details GIT_GARBAGE_COLLECT = False @@ -575,31 +569,6 @@ ALLOW_HTTP_PUSH = False # Path to Gitolite-shell if using that, None to use Git directly HTTP_REPO_ACCESS_GITOLITE = None -# repoSpanner integration settings -# Path the the repoBridge binary -REPOBRIDGE_BINARY = "/usr/libexec/repobridge" -# Whether to create new repositories on repoSpanner by default. -# Either None or a region name. -REPOSPANNER_NEW_REPO = None -# Whether to allow admins to override region selection on creation. -REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE = False -# Whether to create new forks on repoSpanner. -# Either None (no repoSpanner), True (same as origin project) or a region name. -REPOSPANNER_NEW_FORK = True -# Whether to allow an admin to manually migrate an individual project. -REPOSPANNER_ADMIN_MIGRATION = False -# The repoSpanner regions to be used in this Pagure instance. -# Example entry: -# 'default': {'url': 'https://nodea.regiona.repospanner.local:8444', -# 'repo_prefix': 'pagure/', -# 'hook': None, -# 'ca': '', -# 'admin_cert': {'cert': '', -# 'key': ''}, -# 'push_cert': {'cert': '', -# 'key': ''}} -REPOSPANNER_REGIONS = {} - # Configuration for the key helper # Look a username up in the database, overrides SSH_KEYS_USERNAME_EXPECT SSH_KEYS_USERNAME_LOOKUP = False @@ -618,30 +587,7 @@ SSH_KEYS_OPTIONS = ( SSH_ADMIN_TOKEN = None # ACL Checker options -SSH_COMMAND_REPOSPANNER = ( - [ - "/usr/libexec/repobridge", - "--extra", - "username", - "%(username)s", - "--extra", - "repotype", - "%(repotype)s", - "--extra", - "project_name", - "%(project_name)s", - "--extra", - "project_user", - "%(project_user)s", - "--extra", - "project_namespace", - "%(project_namespace)s", - "%(cmd)s", - "'%(repospanner_reponame)s'", - ], - {"REPOBRIDGE_CONFIG": "/etc/repospanner/bridge_%(region)s.json"}, -) -SSH_COMMAND_NON_REPOSPANNER = ( +SSH_COMMAND = ( [ "/usr/bin/%(cmd)s", os.path.join(GIT_FOLDER, "%(reponame)s"), diff --git a/pagure/forms.py b/pagure/forms.py index ec21983..c8f586f 100644 --- a/pagure/forms.py +++ b/pagure/forms.py @@ -192,19 +192,6 @@ class ProjectForm(ProjectFormSimplified): [wtforms.validators.optional()], false_values=FALSE_VALUES, ) - repospanner_region = wtforms.SelectField( - "repoSpanner Region", - [wtforms.validators.optional()], - choices=( - [("none", "Disabled")] - + [ - (region, region) - for region in pagure_config["REPOSPANNER_REGIONS"].keys() - ] - ), - coerce=convert_value, - default=pagure_config["REPOSPANNER_NEW_REPO"], - ) default_branch = wtforms.StringField( "Default branch", [wtforms.validators.optional()], @@ -241,12 +228,6 @@ class ProjectForm(ProjectFormSimplified): ): self.ignore_existing_repos = None - if not ( - is_admin() - and pagure_config.get("REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE") - ): - self.repospanner_region = None - class IssueFormSimplied(PagureForm): """Form to create or edit an issue.""" diff --git a/pagure/hooks/__init__.py b/pagure/hooks/__init__.py index d49fcda..c31e935 100644 --- a/pagure/hooks/__init__.py +++ b/pagure/hooks/__init__.py @@ -157,11 +157,6 @@ class BaseHook(object): """Install the generic post-receive hook that allow us to call multiple post-receive hooks as set per plugin. """ - if project.is_on_repospanner: - # If the project is on repoSpanner, there's nothing to set up, - # as the hook script will be arranged by repo creation. - return - hook_files = os.path.join( os.path.dirname(os.path.realpath(__file__)), "files" ) @@ -297,8 +292,7 @@ def run_project_hooks( """Function to run the hooks on a project This will first call all the plugins with a Runner on the project, - and afterwards, for a non-repoSpanner repo, run all hooks/.* - scripts in the repo. + and afterwards run all hooks/.* scripts in the repo. Args: session: Database session @@ -398,10 +392,6 @@ def run_project_hooks( print(str(e)) haderrors = True - if project.is_on_repospanner: - # We are done. We are not doing any legacy hooks for repoSpanner - return - hookdir = os.path.join(repodir, "hooks") if not os.path.exists(hookdir): return diff --git a/pagure/hooks/default.py b/pagure/hooks/default.py index 5581ff8..2074f1e 100644 --- a/pagure/hooks/default.py +++ b/pagure/hooks/default.py @@ -473,9 +473,7 @@ class DefaultRunner(BaseRunner): but_uids=pr_uids, ) - if not project.is_on_repospanner and _config.get( - "GIT_GARBAGE_COLLECT", False - ): + if _config.get("GIT_GARBAGE_COLLECT", False): pagure.lib.tasks.git_garbage_collect.delay( project.repopath("main") ) diff --git a/pagure/hooks/files/repospannerhook b/pagure/hooks/files/repospannerhook deleted file mode 100755 index 80e0a90..0000000 --- a/pagure/hooks/files/repospannerhook +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - (c) 2018 - Copyright Red Hat Inc - - Authors: - Patrick Uiterwijk - -""" -from __future__ import print_function, absolute_import - -import os -import sys - -if os.environ.get("extra_internal_no_hooks", False): - # we do this check before any pagure machinery is imported - # and initialized to make sure this is very fast (used - # when pushing code from original repos to forks) - sys.exit(0) - -# These fields get filled in by upload-repospanner-hooks -os.environ["PAGURE_CONFIG"] = "${config}" -PYPATH = "${pypath}" - -# Prepare code imports -if PYPATH: - sys.path.insert(0, PYPATH) - -import pagure -import pagure.lib.model_base -import pagure.lib.query -from pagure.hooks import run_project_hooks, extract_changes -from pagure.config import config as pagure_config - - -# Get information from the environment -hooktype = os.path.basename(sys.argv[0]) - -is_internal = os.environ.get("extra_internal", False) == "yes" -pushuser = os.environ["extra_username"] -repotype = os.environ["extra_repotype"] -project_name = os.environ["extra_project_name"] -project_user = os.environ.get("extra_project_user", None) or None -project_namespace = os.environ.get("extra_project_namespace", None) or None -pruid = os.environ.get("extra_pull_request_uid", None) - -changes = extract_changes(from_stdin=hooktype != "update") - -session = pagure.lib.model_base.create_session(pagure_config["DB_URL"]) -if not session: - raise Exception("Unable to initialize db session") - -gitdir = os.path.abspath(os.environ["GIT_DIR"]) - -project = pagure.lib.query._get_project( - session, project_name, project_user, project_namespace -) -if not project: - print("No project found") - sys.exit(1) - -pull_request = None -if pruid: - pull_request = pagure.lib.query.get_request_by_uid( - session, pruid - ) - -run_project_hooks( - session, - pushuser, - project, - hooktype, - repotype, - gitdir, - changes, - is_internal, - pull_request, -) diff --git a/pagure/internal/__init__.py b/pagure/internal/__init__.py index ca29633..d445998 100644 --- a/pagure/internal/__init__.py +++ b/pagure/internal/__init__.py @@ -209,12 +209,8 @@ def check_ssh_access(): { "access": True, "reponame": gitdir, - "repospanner_reponame": project._repospanner_repo_name(repotype) - if project.is_on_repospanner - else None, "repopath": path, "repotype": repotype, - "region": project.repospanner_region, "project_name": project.name, "project_user": project.user.username if project.is_fork else None, "project_namespace": project.namespace, diff --git a/pagure/lib/git.py b/pagure/lib/git.py index efa8a5e..377bf7f 100644 --- a/pagure/lib/git.py +++ b/pagure/lib/git.py @@ -21,7 +21,6 @@ import zipfile import arrow import pygit2 -import requests import six # from sqlalchemy.orm.session import Session @@ -438,7 +437,6 @@ def get_project_from_json(session, jsondata): user=user.username, name=name, namespace=namespace, - repospanner_region=None, description=jsondata.get("description"), parent_id=parent.id if parent else None, blacklist=pagure_config.get("BLACKLISTED_PROJECTS", []), @@ -1008,62 +1006,22 @@ class TemporaryClone(object): if self._parent: self.repopath = os.path.join(self.repopath, self._parent) os.makedirs(self.repopath) - if not self._project.is_on_repospanner: - # This is the simple case. Just do a local clone - # use either the specified path or the use the path of the - # specified project - self._origpath = self._path or self._project.repopath( - self._repotype - ) - if self._origpath is None: - # No repository of this type - # 'main' is already caught and returns an error in repopath() - return None - if not os.path.exists(self._origpath): - return None - PagureRepo.clone(self._origpath, self.repopath) - # Because for whatever reason, one pygit2.Repository is not - # equal to another.... The pygit2.Repository returned from - # pygit2.clone_repository does not have the "branches" attribute. - self.repo = pygit2.Repository(self.repopath) - self._origrepo = pygit2.Repository(self._origpath) - else: - repourl, regioninfo = self._project.repospanner_repo_info( - self._repotype - ) - - command = [ - "git", - "-c", - "protocol.ext.allow=always", - "clone", - "ext::%s %s" - % ( - pagure_config["REPOBRIDGE_BINARY"], - self._project._repospanner_repo_name(self._repotype), - ), - self.repopath, - ] - environ = os.environ.copy() - environ.update( - { - "USER": "pagure", - "REPOBRIDGE_CONFIG": ":environment:", - "REPOBRIDGE_BASEURL": regioninfo["url"], - "REPOBRIDGE_CA": regioninfo["ca"], - "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"], - "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"], - } - ) - with open(os.devnull, "w") as devnull: - subprocess.check_call( - command, - stdout=devnull, - stderr=subprocess.STDOUT, - env=environ, - ) - self.repo = pygit2.Repository(self.repopath) - self._origrepo = self.repo + # This is the simple case. Just do a local clone + # use either the specified path or the use the path of the + # specified project + self._origpath = self._path or self._project.repopath(self._repotype) + if self._origpath is None: + # No repository of this type + # 'main' is already caught and returns an error in repopath() + return None + if not os.path.exists(self._origpath): + return None + PagureRepo.clone(self._origpath, self.repopath) + # Because for whatever reason, one pygit2.Repository is not + # equal to another.... The pygit2.Repository returned from + # pygit2.clone_repository does not have the "branches" attribute. + self.repo = pygit2.Repository(self.repopath) + self._origrepo = pygit2.Repository(self._origpath) # Make sure that all remote refs are mapped to local ones. headname = None @@ -1101,10 +1059,7 @@ class TemporaryClone(object): this TemporaryClone instance with """ self._project = new_project - if not new_project.is_on_repospanner: - self.repo.remotes.set_push_url( - "origin", new_project.repopath("main") - ) + self.repo.remotes.set_push_url("origin", new_project.repopath("main")) def mirror(self, username, force=False, **extra): """Run ``git push --mirror`` of the repo to its origin. @@ -1147,52 +1102,10 @@ class TemporaryClone(object): extra["pull_request_uid"] = extra["pull_request"].uid del extra["pull_request"] - if self._project.is_on_repospanner: - regioninfo = pagure_config["REPOSPANNER_REGIONS"][ - self._project.repospanner_region - ] - - extra.update( - { - "username": username, - "repotype": self._repotype, - "project_name": self._project.name, - "project_user": self._project.user.username - if self._project.is_fork - else "", - "project_namespace": self._project.namespace or "", - } - ) - args = [] - for opt in extra: - args.extend(["--extra", opt, extra[opt]]) - command = [ - "git", - "-c", - "protocol.ext.allow=always", - "push", - "ext::%s %s %s" - % ( - pagure_config["REPOBRIDGE_BINARY"], - " ".join(args), - self._project._repospanner_repo_name(self._repotype), - ), - "--repo", - self.repopath, - ] - environ = { - "USER": "pagure", - "REPOBRIDGE_CONFIG": ":environment:", - "REPOBRIDGE_BASEURL": regioninfo["url"], - "REPOBRIDGE_CA": regioninfo["ca"], - "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"], - "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"], - } - else: - command = ["git", "push", "origin"] - if force: - command.append("--force") - environ = {} + command = ["git", "push", "origin"] + if force: + command.append("--force") + environ = {} command.append("--follow-tags") @@ -2729,198 +2642,91 @@ def drop_branch_aliases(project, source, dest): def delete_project_repos(project): - """Deletes the actual git repositories on disk or repoSpanner + """Deletes the actual git repositories on disk Args: project (Project): Project to delete repos for """ for repotype in pagure.lib.query.get_repotypes(): - if project.is_on_repospanner: - _, regioninfo = project.repospanner_repo_info(repotype) + repopath = project.repopath(repotype) + if repopath is None: + continue - _log.debug("Deleting repotype %s", repotype) - data = { - "Reponame": project._repospanner_repo_name( - repotype, project.repospanner_region - ) - } - resp = requests.post( - "%s/admin/deleterepo" % regioninfo["url"], - json=data, - verify=regioninfo["ca"], - cert=( - regioninfo["admin_cert"]["cert"], - regioninfo["admin_cert"]["key"], - ), + try: + shutil.rmtree(repopath) + except Exception: + _log.exception( + "Failed to remove repotype %s for %s", + repotype, + project.fullname, ) - resp.raise_for_status() - resp = resp.json() - _log.debug("Response json: %s", resp) - if not resp["Success"]: - raise Exception( - "Error in repoSpanner API call: %s" % resp["Error"] - ) - - else: - repopath = project.repopath(repotype) - if repopath is None: - continue - - try: - shutil.rmtree(repopath) - except Exception: - _log.exception( - "Failed to remove repotype %s for %s", - repotype, - project.fullname, - ) -def set_up_project_hooks(project, region, hook=None): +def set_up_project_hooks(project, hook=None): """Makes sure the git repositories for a project have their hooks setup. Args: project (model.Project): Project to set up hooks for - region (string or None): repoSpanner region to set hooks up for - hook (string): The hook ID to set up in repoSpanner (tests only) """ - if region is None: - # This repo is not on repoSpanner, create hooks locally - pagure.hooks.BaseHook.set_up(project) - else: - regioninfo = pagure_config["REPOSPANNER_REGIONS"].get(region) - if not regioninfo: - raise ValueError( - "Invalid repoSpanner region %s looked up" % region - ) - if not hook: - hook = regioninfo["hook"] - if not hook: - # No hooks to set up for this region - return - - for repotype in pagure.lib.query.get_repotypes(): - data = { - "Reponame": project._repospanner_repo_name(repotype, region), - "UpdateRequest": { - "hook-prereceive": hook, - "hook-update": hook, - "hook-postreceive": hook, - }, - } - resp = requests.post( - "%s/admin/editrepo" % regioninfo["url"], - json=data, - verify=regioninfo["ca"], - cert=( - regioninfo["admin_cert"]["cert"], - regioninfo["admin_cert"]["key"], - ), - ) - resp.raise_for_status() - resp = resp.json() - _log.debug("Response json: %s", resp) - if not resp["Success"]: - raise Exception( - "Error in repoSpanner API call: %s" % resp["Error"] - ) + # Create hooks locally + pagure.hooks.BaseHook.set_up(project) -def _create_project_repo(project, region, templ, ignore_existing, repotype): - """Creates a single specific git repository on disk or repoSpanner +def _create_project_repo(project, templ, ignore_existing, repotype): + """Creates a single specific git repository on disk Args: project (Project): Project to create repos for - region (string or None): repoSpanner region to create the repos in - templ (string): Template directory, only valid for non-repoSpanner + templ (string): Template directory ignore_existing (bool): Whether a repo already existing is fatal repotype (string): Repotype to create Returns: (string or None): Directory created """ - if region: - # repoSpanner creation - regioninfo = pagure_config["REPOSPANNER_REGIONS"][region] - # First create the repository on the repoSpanner region - _log.debug("Creating repotype %s", repotype) - data = { - "Reponame": project._repospanner_repo_name(repotype, region), - "Public": False, - } - resp = requests.post( - "%s/admin/createrepo" % regioninfo["url"], - json=data, - verify=regioninfo["ca"], - cert=( - regioninfo["admin_cert"]["cert"], - regioninfo["admin_cert"]["key"], - ), - ) - resp.raise_for_status() - resp = resp.json() - _log.debug("Response json: %s", resp) - if not resp["Success"]: - if "already exists" in resp["Error"]: - if ignore_existing: - return None - else: - raise pagure.exceptions.RepoExistsException(resp["Error"]) - raise Exception( - "Error in repoSpanner API call: %s" % resp["Error"] - ) - + repodir = project.repopath(repotype) + if repodir is None: + # This repo type is disabled return None - - else: - # local repo - repodir = project.repopath(repotype) - if repodir is None: - # This repo type is disabled + if os.path.exists(repodir): + if not ignore_existing: + raise pagure.exceptions.RepoExistsException( + "The %s repo %s already exists" % (repotype, project.path) + ) + else: return None - if os.path.exists(repodir): - if not ignore_existing: - raise pagure.exceptions.RepoExistsException( - "The %s repo %s already exists" % (repotype, project.path) - ) - else: - return None - if repotype == "main": - pygit2.init_repository(repodir, bare=True, template_path=templ) + if repotype == "main": + pygit2.init_repository(repodir, bare=True, template_path=templ) - if not project.private: - # Make the repo exportable via apache - http_clone_file = os.path.join(repodir, "git-daemon-export-ok") - if not os.path.exists(http_clone_file): - with open(http_clone_file, "w"): - pass - else: - pygit2.init_repository( - repodir, - bare=True, - mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP, - ) + if not project.private: + # Make the repo exportable via apache + http_clone_file = os.path.join(repodir, "git-daemon-export-ok") + if not os.path.exists(http_clone_file): + with open(http_clone_file, "w"): + pass + else: + pygit2.init_repository( + repodir, + bare=True, + mode=pygit2.C.GIT_REPOSITORY_INIT_SHARED_GROUP, + ) - return repodir + return repodir -def create_project_repos(project, region, templ, ignore_existing): - """Creates the actual git repositories on disk or repoSpanner +def create_project_repos(project, templ, ignore_existing): + """Creates the actual git repositories on disk Args: project (Project): Project to create repos for - region (string or None): repoSpanner region to create the repos in - templ (string): Template directory, only valid for non-repoSpanner + templ (string): Template directory ignore_existing (bool): Whether a repo already existing is fatal """ - if region and templ: - raise Exception("repoSpanner is incompatible with template directory") - created_dirs = [] try: for repotype in pagure.lib.query.get_repotypes(): created = _create_project_repo( - project, region, templ, ignore_existing, repotype + project, templ, ignore_existing, repotype ) if created: created_dirs.append(created) @@ -2929,7 +2735,7 @@ def create_project_repos(project, region, templ, ignore_existing): shutil.rmtree(created) raise - set_up_project_hooks(project, region) + set_up_project_hooks(project) def get_stats_patch(patch): @@ -3123,50 +2929,8 @@ def mirror_pull_project(session, project, debug=False): if debug: print("Pushing to the local git repo") extra = {} - if project.is_on_repospanner: - regioninfo = pagure_config["REPOSPANNER_REGIONS"][ - project.repospanner_region - ] - - extra.update( - { - "username": "pagure", - "repotype": "main", - "project_name": project.name, - "project_user": project.user.username - if project.is_fork - else "", - "project_namespace": project.namespace or "", - } - ) - args = [] - for opt in extra: - args.extend(["--extra", opt, extra[opt]]) - command = [ - "git", - "-c", - "protocol.ext.allow=always", - "push", - "ext::%s %s %s" - % ( - pagure_config["REPOBRIDGE_BINARY"], - " ".join(args), - project._repospanner_repo_name("main"), - ), - "--repo", - repopath, - ] - environ = { - "USER": "pagure", - "REPOBRIDGE_CONFIG": ":environment:", - "REPOBRIDGE_BASEURL": regioninfo["url"], - "REPOBRIDGE_CA": regioninfo["ca"], - "REPOBRIDGE_CERT": regioninfo["push_cert"]["cert"], - "REPOBRIDGE_KEY": regioninfo["push_cert"]["key"], - } - else: - command = ["git", "push", "local", "--mirror"] - environ = {} + command = ["git", "push", "local", "--mirror"] + environ = {} _log.debug("Running a git push to %s", project.fullname) env = os.environ.copy() diff --git a/pagure/lib/model.py b/pagure/lib/model.py index 7045f5e..4c63aef 100644 --- a/pagure/lib/model.py +++ b/pagure/lib/model.py @@ -20,7 +20,6 @@ import re from operator import attrgetter import arrow -import pygit2 import six import sqlalchemy as sa from sqlalchemy import create_engine @@ -422,7 +421,6 @@ class Project(BASE): backref="projects", ) private = sa.Column(sa.Boolean, nullable=False, default=False) - repospanner_region = sa.Column(sa.Text, nullable=True) users = relation( "User", @@ -538,87 +536,12 @@ class Project(BASE): return "%s-project-%s" % (self.fullname, self.id) @property - def is_on_repospanner(self): - """Returns whether this repo is on repoSpanner.""" - return self.repospanner_region is not None - - @property def path(self): """Return the name of the git repo on the filesystem.""" return "%s.git" % self.fullname - def repospanner_repo_info(self, repotype, region=None): - """Returns info for getting a repoSpanner repo for a project. - - Args: - repotype (string): Type of repository - region (string): If repo is not on repoSpanner, return url as if - it was in this region. Used for migrating to repoSpanner. - Return type: (url, dict): First is the clone url, then a dict with - the regioninfo. - """ - if not self.is_on_repospanner and region is None: - raise ValueError("Repo %s is not on repoSpanner" % self.fullname) - if self.is_on_repospanner and region is not None: - raise ValueError( - "Repo %s is already on repoSpanner" % self.fullname - ) - if region is None: - region = self.repospanner_region - regioninfo = pagure_config["REPOSPANNER_REGIONS"].get(region) - if not regioninfo: - raise ValueError( - "Invalid repoSpanner region %s looked up" % region - ) - - url = "%s/repo/%s.git" % ( - regioninfo["url"], - self._repospanner_repo_name(repotype, region), - ) - return url, regioninfo - - def _repospanner_repo_name(self, repotype, region=None): - """Returns the name of a repo as named in repoSpanner. - - Args: - repotype (string): Type of repository - region (string): repoSpanner region name - Return type: (string) - """ - if region is None: - region = self.repospanner_region - return os.path.join( - pagure_config["REPOSPANNER_REGIONS"][region].get( - "repo_prefix", "" - ), - repotype, - self.fullname, - ) - def repopath(self, repotype): - """Return the full repository path of the git repo on the filesystem. - - If the repository is on repoSpanner, this will be a pseudo repository, - which is "git repo enough" to be considered a valid repo, but any - access should go through a repoSpanner enlightened libgit2. - """ - if self.is_on_repospanner: - pseudopath = os.path.join( - pagure_config["REPOSPANNER_PSEUDO_FOLDER"], repotype, self.path - ) - if not os.path.exists(pseudopath): - repourl, regioninfo = self.repospanner_repo_info(repotype) - fake = pygit2.init_repository(pseudopath, bare=True) - fake.config["repospanner.url"] = repourl - fake.config["repospanner.cert"] = regioninfo["push_cert"][ - "cert" - ] - fake.config["repospanner.key"] = regioninfo["push_cert"]["key"] - fake.config["repospanner.cacert"] = regioninfo["ca"] - fake.config["repospanner.enabled"] = True - del fake - return pseudopath - + """Return the full repository path of the git repo on the filesystem.""" maindir = None if repotype == "main": maindir = pagure_config["GIT_FOLDER"] diff --git a/pagure/lib/query.py b/pagure/lib/query.py index a7a03ce..ae5b3ab 100644 --- a/pagure/lib/query.py +++ b/pagure/lib/query.py @@ -1650,7 +1650,6 @@ def new_project( name, blacklist, allowed_prefix, - repospanner_region, description=None, url=None, avatar_email=None, @@ -1716,21 +1715,9 @@ def new_project( 'It is not possible to create the repo "%s"' % (path) ) - if repospanner_region == "none": - repospanner_region = None - elif repospanner_region is None: - repospanner_region = pagure_config["REPOSPANNER_NEW_REPO"] - - if ( - repospanner_region - and repospanner_region not in pagure_config["REPOSPANNER_REGIONS"] - ): - raise Exception("repoSpanner region %s invalid" % repospanner_region) - project = model.Project( name=name, namespace=namespace, - repospanner_region=repospanner_region, description=description if description else None, url=url if url else None, avatar_email=avatar_email if avatar_email else None, @@ -2336,19 +2323,10 @@ def fork_project(session, user, repo, editbranch=None, editfile=None): user_obj = get_user(session, user) - fork_repospanner_setting = pagure_config["REPOSPANNER_NEW_FORK"] - if fork_repospanner_setting is None: - repospanner_region = None - elif fork_repospanner_setting is True: - repospanner_region = repo.repospanner_region - else: - repospanner_region = fork_repospanner_setting - project = model.Project( name=repo.name, namespace=repo.namespace, description=repo.description, - repospanner_region=repospanner_region, private=repo.private, user_id=user_obj.id, parent_id=repo.id, diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 59000ad..0f32b19 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -15,7 +15,6 @@ import datetime import hashlib import os import os.path -import shutil import subprocess import time @@ -276,7 +275,6 @@ def create_project( with project.lock("WORKER"): pagure.lib.git.create_project_repos( project, - project.repospanner_region, templ, ignore_existing_repo, ) @@ -484,9 +482,7 @@ def fork( ) with repo_to.lock("WORKER"): - pagure.lib.git.create_project_repos( - repo_to, repo_to.repospanner_region, None, False - ) + pagure.lib.git.create_project_repos(repo_to, None, False) with pagure.lib.git.TemporaryClone( repo_from, "main", "fork" @@ -506,14 +502,14 @@ def fork( tempclone.change_project_association(repo_to) tempclone.mirror("pagure", internal_no_hooks="yes") - if not repo_to.is_on_repospanner and not repo_to.private: + if not repo_to.private: # Create the git-daemon-export-ok file on the clone http_clone_file = os.path.join( repo_to.repopath("main"), "git-daemon-export-ok" ) - if not os.path.exists(http_clone_file): - with open(http_clone_file, "w"): - pass + if not os.path.exists(http_clone_file): + with open(http_clone_file, "w"): + pass # Finally set the default branch to be the same as the parent repo_from_obj = pygit2.Repository(repo_from.repopath("main")) @@ -607,105 +603,6 @@ def refresh_remote_pr(self, session, name, namespace, user, requestid): @conn.task(queue=pagure_config.get("FAST_CELERY_QUEUE", None), bind=True) @pagure_task -def move_to_repospanner(self, session, name, namespace, user, region): - """Move a repository to a repoSpanner region.""" - project = pagure.lib.query._get_project( - session, namespace=namespace, name=name, user=user - ) - regioninfo = pagure_config.get("REPOSPANNER_REGIONS", {}).get(region) - if not regioninfo: - raise Exception("Missing region config") - - with project.lock("WORKER"): - # Perform some pre-flight checks - if project.is_on_repospanner: - raise Exception("Project is already on repoSpanner") - - # Make sure that no non-runner hooks are enabled for this project - incompatible_hooks = [] - for repotype in pagure.lib.query.get_repotypes(): - path = project.repopath(repotype) - if path is None: - continue - hookpath = os.path.join(path, "hooks") - for hook in os.listdir(hookpath): - if not hook.startswith( - ("pre-receive.", "update.", "post-receive.") - ): - continue - if hook.endswith(".sample"): - # Ignore the samples that Git inserts - continue - if hook not in pagure.lib.query.ORIGINAL_PAGURE_HOOK: - incompatible_hooks.append((repotype, hook)) - - if incompatible_hooks: - raise Exception( - "Repository contains repoSpanner-incompatible " - "hooks: %s" - % ", ".join(["%s" % (hook,) for hook in incompatible_hooks]) - ) - - # Create the repositories - pagure.lib.git.create_project_repos(project, region, None, False) - - for repotype in pagure.lib.query.get_repotypes(): - repopath = project.repopath(repotype) - if repopath is None: - continue - repourl, _ = project.repospanner_repo_info(repotype, region) - repo_obj = pagure.lib.repo.PagureRepo(repopath) - repo_obj.create_remote("repospanner_push", repourl) - - command = [ - "git", - "-c", - "http.sslcainfo=%s" % regioninfo["ca"], - "-c", - "http.sslcert=%s" % regioninfo["push_cert"]["cert"], - "-c", - "http.sslkey=%s" % regioninfo["push_cert"]["key"], - "push", - "--mirror", - "repospanner_push", - ] - _log.debug("Running push command: %s", command) - out = subprocess.check_output( - command, cwd=repopath, stderr=subprocess.STDOUT - ) - _log.debug("Out: %s" % out) - - for repotype in pagure.lib.query.get_repotypes(): - repopath = project.repopath(repotype) - if repopath is None: - continue - repo_obj = pagure.lib.repo.PagureRepo(repopath) - - # At this moment, this subrepo has been migrated - # Move the "refs" folder to "refsold", so that we don't actually - # delete any data, but it's no longer a valid git repo. - # On next use, a pseudo repository will be created. - refsdir = os.path.join(repopath, "refs") - shutil.move(refsdir, refsdir + "old") - with open( - os.path.join(repopath, "repospanner_status"), "w" - ) as info: - info.write( - "This repository has migrated to repoSpanner region %s" - % region - ) - - project.repospanner_region = region - session.add(project) - session.commit() - - return ret( - "ui_ns.view_repo", username=user, namespace=namespace, repo=name - ) - - -@conn.task(queue=pagure_config.get("FAST_CELERY_QUEUE", None), bind=True) -@pagure_task def refresh_pr_cache(self, session, name, namespace, user, but_uids=None): """Refresh the merge status cached of pull-requests.""" project = pagure.lib.query._get_project( diff --git a/pagure/templates/new_project.html b/pagure/templates/new_project.html index bdce2d5..194781b 100644 --- a/pagure/templates/new_project.html +++ b/pagure/templates/new_project.html @@ -28,9 +28,6 @@ {{ render_bootstrap_field(form.mirrored_from, field_description="Mirror this project from another git server") }} {% endif %} {{ render_bootstrap_field(form.create_readme, field_description="Create a README file automatically") }} - {% if form.repospanner_region %} - {{ render_bootstrap_field(form.repospanner_region, field_description="repoSpanner region to create the project in") }} - {% endif %} diff --git a/pagure/templates/settings.html b/pagure/templates/settings.html index f235e0c..1719dc0 100644 --- a/pagure/templates/settings.html +++ b/pagure/templates/settings.html @@ -93,11 +93,6 @@ href="#giveproject-tab" role="tab" aria-controls="giveproject">Give Project {% endif %} - {% if config.get('REPOSPANNER_REGIONS') %} - repoSpanner state - {% endif %} - {% if (not repo.is_fork and config.get('ENABLE_DEL_PROJECTS', True)) or (repo.is_fork and config.get('ENABLE_DEL_FORKS', @@ -1064,12 +1059,6 @@ {% endif %} - {% if config.get('REPOSPANNER_REGIONS') %} -
- {% include 'settings_repospanner.html' %} -
- {% endif %} - {% if (not repo.is_fork and config.get('ENABLE_DEL_PROJECTS', True)) or diff --git a/pagure/templates/settings_repospanner.html b/pagure/templates/settings_repospanner.html deleted file mode 100644 index d56cad0..0000000 --- a/pagure/templates/settings_repospanner.html +++ /dev/null @@ -1,43 +0,0 @@ -

- repoSpanner status -

-
-
-

- repoSpanner is - a distributed git storage server allowing to keep in sync multiple git - repositories in multiple locations. -

-

- {% if repo.is_on_repospanner %} - This repository is on repoSpanner region {{ repo.repospanner_region }} - {% else %} - This repository is NOT yet on repoSpanner - {% if pagure_admin and config.get('REPOSPANNER_ADMIN_MIGRATION') %} -

-
-
-
-
-
- {{ tag_form.csrf_token }} - Select region to move to: - - -
- {% endif %} - {% endif %} -
-
diff --git a/pagure/ui/app.py b/pagure/ui/app.py index bf4694c..7c9f2d6 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -1084,10 +1084,6 @@ def new_project(): namespace = form.namespace.data if namespace: namespace = namespace.strip() - if form.repospanner_region: - repospanner_region = form.repospanner_region.data - else: - repospanner_region = None if form.ignore_existing_repos: ignore_existing_repos = form.ignore_existing_repos.data else: @@ -1108,7 +1104,6 @@ def new_project(): private=private, description=description, namespace=namespace, - repospanner_region=repospanner_region, url=url, avatar_email=avatar_email, user=flask.g.fas_user.username, diff --git a/pagure/ui/clone.py b/pagure/ui/clone.py index ab4fbbc..81aba1f 100644 --- a/pagure/ui/clone.py +++ b/pagure/ui/clone.py @@ -17,7 +17,6 @@ import subprocess import tempfile import flask -import requests import werkzeug.wsgi import pagure.exceptions @@ -94,11 +93,7 @@ def _get_remote_user(project): def proxy_raw_git(project): - """Proxy a request to Git or gitolite3 via a subprocess. - - This should get called after it is determined the requested project - is not on repoSpanner. - """ + """Proxy a request to Git or gitolite3 via a subprocess.""" _log.debug("Raw git clone proxy started") remote_user = _get_remote_user(project) # We are going to shell out to gitolite-shell. Prepare the env it needs. @@ -226,63 +221,6 @@ def proxy_raw_git(project): ) -def proxy_repospanner(project, service): - """Proxy a request to repoSpanner. - - Args: - project (model.Project): The project being accessed - service (String): The service as indicated by ?Service= in /info/refs - """ - oper = os.path.basename(flask.request.path) - if oper == "refs": - oper = "info/refs?service=%s" % service - regionurl, regioninfo = project.repospanner_repo_info("main") - url = "%s/%s" % (regionurl, oper) - - # Older flask/werkzeug versions don't support both an input and output - # stream: this results in a blank upload. - # So, we optimize for the direction the majority of the data will likely - # flow. - streamargs = {} - if service == "git-receive-pack": - # This is a Push operation, optimize for data from the client - streamargs["data"] = flask.request.stream - streamargs["stream"] = False - else: - # This is a Pull operation, optimize for data from the server - streamargs["data"] = flask.request.data - streamargs["stream"] = True - - resp = requests.request( - flask.request.method, - url, - verify=regioninfo["ca"], - cert=(regioninfo["push_cert"]["cert"], regioninfo["push_cert"]["key"]), - headers={ - "Content-Encoding": flask.request.content_encoding, - "Content-Type": flask.request.content_type, - "X-Extra-Username": flask.request.remote_user, - "X-Extra-Repotype": "main", - "X-Extra-project_name": project.name, - "x-Extra-project_user": project.user if project.is_fork else "", - "X-Extra-project_namespace": project.namespace, - }, - **streamargs, - ) - - # Strip out any headers that cause problems - for name in ("transfer-encoding",): - if name in resp.headers: - del resp.headers[name] - - return flask.Response( - resp.iter_content(chunk_size=128), - status=resp.status_code, - headers=dict(resp.headers), - direct_passthrough=True, - ) - - def clone_proxy(project, username=None, namespace=None): """Proxy the /info/refs endpoint for HTTP pull/push. @@ -419,10 +357,7 @@ def clone_proxy(project, username=None, namespace=None): ) flask.abort(404, description="Project not found") - if project_obj.is_on_repospanner: - return proxy_repospanner(project_obj, service) - else: - return proxy_raw_git(project_obj) + return proxy_raw_git(project_obj) def add_clone_proxy_cmds(): diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index c3c4b1d..535ef72 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -3096,71 +3096,6 @@ def delete_report(repo, username=None, namespace=None): ) -@UI_NS.route("//torepospanner", methods=["POST"]) -@UI_NS.route("///torepospanner", methods=["POST"]) -@UI_NS.route("/fork///torepospanner", methods=["POST"]) -@UI_NS.route( - "/fork////torepospanner", methods=["POST"] -) -@login_required -@is_admin_sess_timedout -@is_repo_admin -def move_to_repospanner(repo, username=None, namespace=None): - """Give a project to someone else.""" - repo = flask.g.repo - - if not pagure.utils.is_admin(): - flask.abort( - 403, - description="You are not allowed to transfer this project " - "to repoSpanner", - ) - - if not pagure_config.get("REPOSPANNER_ADMIN_MIGRATION"): - flask.abort( - 403, description="It is not allowed to request migration of a repo" - ) - - form = pagure.forms.ConfirmationForm() - - if form.validate_on_submit(): - region = flask.request.form.get("region", "").strip() - if not region: - flask.abort(404, description="No target region specified") - - if region not in pagure_config.get("REPOSPANNER_REGIONS"): - flask.abort(404, description="Invalid region specified") - - _log.info( - "Repo %s requested to be migrated to repoSpanner region %s", - repo.fullname, - region, - ) - - task = pagure.lib.tasks.move_to_repospanner.delay( - repo.name, namespace, username, region - ) - - return pagure.utils.wait_for_task( - task, - prev=flask.url_for( - "ui_ns.view_repo", - username=username, - repo=repo.name, - namespace=namespace, - ), - ) - - return flask.redirect( - flask.url_for( - "ui_ns.view_repo", - username=username, - repo=repo.name, - namespace=namespace, - ) - ) - - @UI_NS.route("//give", methods=["POST"]) @UI_NS.route("///give", methods=["POST"]) @UI_NS.route("/fork///give", methods=["POST"]) diff --git a/tests/__init__.py b/tests/__init__.py index 598ebf5..3bdbf63 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -91,7 +91,6 @@ GIT_FOLDER = '%(path)s/repos' REQUESTS_FOLDER = '%(path)s/repos/requests' TICKETS_FOLDER = %(tickets_folder)r DOCS_FOLDER = %(docs_folder)r -REPOSPANNER_PSEUDO_FOLDER = '%(path)s/repos/pseudo' ATTACHMENTS_FOLDER = '%(path)s/attachments' BROKER_URL = 'redis+socket://%(global_path)s/broker' CELERY_CONFIG = { @@ -100,21 +99,6 @@ CELERY_CONFIG = { } GIT_AUTH_BACKEND = '%(authbackend)s' TEST_AUTH_STATUS = '%(path)s/testauth_status.json' -REPOBRIDGE_BINARY = '%(repobridge_binary)s' -REPOSPANNER_NEW_REPO = %(repospanner_new_repo)s -REPOSPANNER_NEW_REPO_ADMIN_OVERRIDE = %(repospanner_admin_override)s -REPOSPANNER_NEW_FORK = %(repospanner_new_fork)s -REPOSPANNER_ADMIN_MIGRATION = %(repospanner_admin_migration)s -REPOSPANNER_REGIONS = { - 'default': {'url': 'https://repospanner.localhost.localdomain:%(repospanner_gitport)s', - 'repo_prefix': 'pagure/', - 'hook': None, - 'ca': '%(path)s/repospanner/pki/ca.crt', - 'admin_cert': {'cert': '%(path)s/repospanner/pki/admin.crt', - 'key': '%(path)s/repospanner/pki/admin.key'}, - 'push_cert': {'cert': '%(path)s/repospanner/pki/pagure.crt', - 'key': '%(path)s/repospanner/pki/pagure.key'}} -} LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -393,12 +377,6 @@ class SimplePagureTest(unittest.TestCase): "tickets_folder": "%s/repos/tickets" % self.path, "global_path": self.dbfolder, "authbackend": "gitolite3", - "repobridge_binary": "/usr/libexec/repobridge", - "repospanner_gitport": str(8443 + sys.version_info.major), - "repospanner_new_repo": "None", - "repospanner_admin_override": "False", - "repospanner_new_fork": "True", - "repospanner_admin_migration": "False", "nogithooks": False, } config_values.update(self.config_values) diff --git a/tests/test_pagure_flask_internal.py b/tests/test_pagure_flask_internal.py index eae24a5..0359459 100644 --- a/tests/test_pagure_flask_internal.py +++ b/tests/test_pagure_flask_internal.py @@ -2853,17 +2853,9 @@ class PagureFlaskInternaltests(tests.Modeltests): self.assertEqual(result["username"], "deploykey_test_2") self.assertEqual(result["public_key"], project_key) - @patch.dict( - "pagure.config.config", - {"REPOSPANNER_REGIONS": {"region": {"repo_prefix": "prefix"}}}, - ) def test_check_ssh_access(self): """Test the SSH access check endpoint.""" tests.create_projects(self.session) - self.session.query(pagure.lib.model.Project).filter( - pagure.lib.model.Project.name == "test2" - ).update({pagure.lib.model.Project.repospanner_region: "region"}) - self.session.commit() url = "/pv/ssh/checkaccess/" @@ -2884,22 +2876,16 @@ class PagureFlaskInternaltests(tests.Modeltests): runtest("tickets/test.git", "foo", False) self.assertEqual(i1["reponame"], "test.git") - self.assertEqual(i1["repospanner_reponame"], None) self.assertEqual(i1["repotype"], "main") - self.assertEqual(i1["region"], None) self.assertEqual(i1["project_name"], "test") self.assertEqual(i1["project_user"], None) self.assertEqual(i1["project_namespace"], None) self.assertEqual(i1, i2) self.assertEqual(i3["reponame"], "tickets/test.git") - self.assertEqual(i3["repospanner_reponame"], None) self.assertEqual(i3["repotype"], "tickets") - self.assertEqual(i3["region"], None) self.assertEqual(i3["project_name"], "test") self.assertEqual(i3["project_user"], None) self.assertEqual(i3["project_namespace"], None) - self.assertEqual(i4["repospanner_reponame"], "prefix/main/test2") - self.assertEqual(i4["region"], "region") if __name__ == "__main__": diff --git a/tests/test_pagure_flask_ui_repo_slash_name.py b/tests/test_pagure_flask_ui_repo_slash_name.py index 5d505ff..ac09ecd 100644 --- a/tests/test_pagure_flask_ui_repo_slash_name.py +++ b/tests/test_pagure_flask_ui_repo_slash_name.py @@ -108,7 +108,6 @@ class PagureFlaskSlashInNametests(tests.SimplePagureTest): self.session, name="test", namespace="forks", - repospanner_region=None, description="test project forks/test", url="", avatar_email="", @@ -187,7 +186,6 @@ class PagureFlaskSlashInNametests(tests.SimplePagureTest): self.session, name="test", namespace="forks", - repospanner_region=None, description="test project forks/test", url="", avatar_email="", diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py index 3c00e0e..fd8c62c 100644 --- a/tests/test_pagure_lib.py +++ b/tests/test_pagure_lib.py @@ -1553,7 +1553,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="static", - repospanner_region=None, blacklist=["static"], allowed_prefix=[], description="description for static", @@ -1568,7 +1567,6 @@ class PagureLibtests(tests.Modeltests): user="pingou", namespace="space", name="static", - repospanner_region=None, blacklist=["space/*"], allowed_prefix=["space"], description="description for static", @@ -1583,7 +1581,6 @@ class PagureLibtests(tests.Modeltests): user="pingou", name="s" * 40, namespace="pingou", - repospanner_region=None, blacklist=["static"], allowed_prefix=["pingou"], description="description for 40 chars length project", @@ -1597,7 +1594,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1620,7 +1616,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="TestProject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1651,7 +1646,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1671,7 +1665,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1688,7 +1681,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1721,7 +1713,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1741,7 +1732,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1760,7 +1750,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1777,7 +1766,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="pingou/" + "s" * 40, - repospanner_region=None, blacklist=["static"], allowed_prefix=["pingou"], description="description for 40 chars length project", @@ -1806,7 +1794,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -1844,7 +1831,6 @@ class PagureLibtests(tests.Modeltests): user="pingou", name="testproject2", namespace="testns", - repospanner_region=None, blacklist=[], allowed_prefix=["testns"], description="description for testproject2", @@ -2275,7 +2261,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -2355,7 +2340,6 @@ class PagureLibtests(tests.Modeltests): session=self.session, user="pingou", name="testproject", - repospanner_region=None, blacklist=[], allowed_prefix=[], description="description for testproject", @@ -2434,7 +2418,6 @@ class PagureLibtests(tests.Modeltests): user="pingou", name="testproject", namespace="foonamespace", - repospanner_region=None, blacklist=[], allowed_prefix=["foonamespace"], description="description for testproject", diff --git a/tests/test_pagure_repospanner.py b/tests/test_pagure_repospanner.py deleted file mode 100644 index b2ef3a1..0000000 --- a/tests/test_pagure_repospanner.py +++ /dev/null @@ -1,675 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - (c) 2015-2018 - Copyright Red Hat Inc - - Authors: - Patrick Uiterwijk - -""" - -from __future__ import unicode_literals, absolute_import - -import datetime -import functools -import munch -import unittest -import shutil -import subprocess -import sys -import tempfile -import time -import os - -import six -import json -import pygit2 -import requests -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry -from mock import patch, MagicMock - -sys.path.insert( - 0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") -) - -import pagure.lib.query -import pagure.cli.admin -import tests - - -REPOSPANNER_CONFIG_TEMPLATE = """ ---- -ca: - path: %(path)s/repospanner/pki -admin: - url: https://localhost.localdomain:%(gitport)s/ - ca: %(path)s/repospanner/pki/ca.crt - cert: %(path)s/repospanner/pki/admin.crt - key: %(path)s/repospanner/pki/admin.key -storage: - state: %(path)s/repospanner/state - git: - type: tree - clustered: true - directory: %(path)s/repospanner/git -listen: - rpc: 127.0.0.1:%(rpcport)s - http: 127.0.0.1:%(gitport)s -certificates: - ca: %(path)s/repospanner/pki/ca.crt - client: - cert: %(path)s/repospanner/pki/repospanner.localhost.crt - key: %(path)s/repospanner/pki/repospanner.localhost.key - server: - default: - cert: %(path)s/repospanner/pki/repospanner.localhost.crt - key: %(path)s/repospanner/pki/repospanner.localhost.key -hooks: - debug: true - bubblewrap: - enabled: false - unshare: - - net - - ipc - - pid - - uts - share_net: false - mount_proc: true - mount_dev: true - uid: - gid: - hostname: myhostname - bind: - ro_bind: - - - /usr - - /usr - - - %(codepath)s - - %(codepath)s - - - %(path)s - - %(path)s - - - %(crosspath)s - - %(crosspath)s - symlink: - - - usr/lib64 - - /lib64 - - - usr/bin - - /bin - runner: %(hookrunner_bin)s - user: 0 -""" - - -class PagureRepoSpannerTests(tests.Modeltests): - """Tests for repoSpanner integration of pagure""" - - repospanner_binary = None - repospanner_runlog = None - repospanner_proc = None - - def run_cacmd(self, logfile, *args): - """Run a repoSpanner CA command.""" - subprocess.check_call( - [ - self.repospanner_binary, - "--config", - os.path.join(self.path, "repospanner", "config.yml"), - # NEVER use this in a production system! It makes repeatable keys - "ca", - ] - + list(args) - + ["--very-insecure-weak-keys"], - stdout=logfile, - stderr=subprocess.STDOUT, - ) - - def setUp(self): - """set up the environment.""" - possible_paths = ["./repospanner", "/usr/bin/repospanner"] - - for option in possible_paths: - option = os.path.abspath(option) - if os.path.exists(option): - self.repospanner_binary = option - break - - if not self.repospanner_binary: - raise unittest.SkipTest("repoSpanner not found") - - hookrunbins = [ - os.path.join( - os.path.dirname(self.repospanner_binary), "repohookrunner" - ), - os.path.join("/usr", "libexec", "repohookrunner"), - ] - found = False - for hookrunbin in hookrunbins: - if os.path.exists(hookrunbin): - found = True - break - if not found: - raise Exception("repoSpanner found, but repohookrunner not") - - repobridgebins = [ - os.path.join( - os.path.dirname(self.repospanner_binary), "repobridge" - ), - os.path.join("/usr", "libexec", "repobridge"), - ] - found = False - for repobridgebin in repobridgebins: - if os.path.exists(repobridgebin): - found = True - break - if not found: - raise Exception("repoSpanner found, but repobridge not") - - self.config_values["repobridge_binary"] = repobridgebin - - codepath = os.path.normpath( - os.path.join(os.path.dirname(os.path.abspath(__file__)), "../") - ) - - # Only run the setUp() function if we are actually going ahead and run - # this test. The reason being that otherwise, setUp will set up a - # database, but because we "error out" from setUp, the tearDown() - # function never gets called, leaving it behind. - super(PagureRepoSpannerTests, self).setUp() - - # TODO: Find free ports - configvals = { - "path": self.path, - "crosspath": self.dbfolder, - "gitport": 8443 + sys.version_info.major, - "rpcport": 8445 + sys.version_info.major, - "codepath": codepath, - "hookrunner_bin": hookrunbin, - } - - os.mkdir(os.path.join(self.path, "repospanner")) - cfgpath = os.path.join(self.path, "repospanner", "config.yml") - with open(cfgpath, "w") as cfg: - cfg.write(REPOSPANNER_CONFIG_TEMPLATE % configvals) - - with open( - os.path.join(self.path, "repospanner", "keylog"), "w" - ) as keylog: - # Create the CA - self.run_cacmd( - keylog, - "init", - "localdomain", - "--no-name-constraint", - "--random-cn", - ) - # Create the node cert - self.run_cacmd(keylog, "node", "localhost", "repospanner") - # Create the admin cert - self.run_cacmd( - keylog, - "leaf", - "admin", - "--admin", - "--region", - "*", - "--repo", - "*", - ) - # Create the Pagure cert - self.run_cacmd( - keylog, - "leaf", - "pagure", - "--read", - "--write", - "--region", - "*", - "--repo", - "*", - ) - - with open( - os.path.join(self.path, "repospanner", "spawnlog"), "w" - ) as spawnlog: - # Initialize state - subprocess.check_call( - [ - self.repospanner_binary, - "--config", - cfgpath, - "serve", - "--spawn", - ], - stdout=spawnlog, - stderr=subprocess.STDOUT, - ) - - self.repospanner_runlog = open( - os.path.join(self.path, "repospanner", "runlog"), "w+" - ) - - try: - self.repospanner_proc = subprocess.Popen( - [ - self.repospanner_binary, - "--config", - cfgpath, - "serve", - "--debug", - ], - stdout=self.repospanner_runlog, - stderr=subprocess.STDOUT, - ) - except: - # Make sure to clean up repoSpanner, since we did start it - self.tearDown() - raise - - attempts = 0 - while True: - try: - # Wait for the instance to become available - resp = requests.get( - "https://repospanner.localhost.localdomain:%d/" - % configvals["gitport"], - verify=os.path.join( - self.path, "repospanner", "pki", "ca.crt" - ), - cert=( - os.path.join( - self.path, "repospanner", "pki", "pagure.crt" - ), - os.path.join( - self.path, "repospanner", "pki", "pagure.key" - ), - ), - ) - resp.raise_for_status() - - print("repoSpanner identification: %s" % resp.text) - break - except: - if attempts < 5: - attempts += 1 - time.sleep(1) - continue - # Make sure to clean up repoSpanner, since we did start it - self.tearDown() - raise - - # Upload the hook script to repoSpanner - args = munch.Munch({"region": "default"}) - hookid = pagure.cli.admin.do_upload_repospanner_hooks(args) - pagure.config.config["REPOSPANNER_REGIONS"]["default"]["hook"] = hookid - - def tearDown(self): - """Tear down the repoSpanner instance.""" - if self.repospanner_proc: - # Tear down - self.repospanner_proc.terminate() - exitcode = self.repospanner_proc.wait() - if exitcode != 0: - print("repoSpanner exit code: %d" % exitcode) - - if self.repospanner_runlog: - self.repospanner_runlog.close() - - super(PagureRepoSpannerTests, self).tearDown() - - -def print_repospanner_log(fn): - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - try: - return fn(self, *args, **kwargs) - finally: - if self.repospanner_runlog: - self.repospanner_runlog.seek(0, 0) - print("repoSpanner log follows:") - print(self.repospanner_runlog.read()) - - return wrapper - - -class PagureRepoSpannerTestsNewRepoDefault(PagureRepoSpannerTests): - config_values = { - "repospanner_new_repo": "'default'", - "authbackend": "test_auth", - } - - @print_repospanner_log - @patch("pagure.ui.app.admin_session_timedout") - def test_new_project(self, ast): - """Test creating a new repo by default on repoSpanner works.""" - ast.return_value = False - - user = tests.FakeUser(username="foo") - with tests.user_set(self.app.application, user): - output = self.app.get("/new/") - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn("Create new Project", output_text) - - data = { - "name": "project-1", - "description": "Project #1", - "create_readme": "y", - "csrf_token": self.get_csrf(), - } - - output = self.app.post("/new/", data=data, follow_redirects=True) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - '
\nProject #1', output_text - ) - self.assertIn( - "Overview - project-1 - Pagure", output_text - ) - self.assertIn("Added the README", output_text) - - output = self.app.get("/project-1/settings") - self.assertIn( - "This repository is on repoSpanner region default", - output.get_data(as_text=True), - ) - - with tests.user_set( - self.app.application, tests.FakeUser(username="pingou") - ): - # Verify that for forking, Git auth status is ignored (hooks should not be run) - self.set_auth_status(False) - - data = {"csrf_token": self.get_csrf()} - - output = self.app.post( - "/do_fork/project-1", data=data, follow_redirects=True - ) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - '
\nProject #1', output_text - ) - self.assertIn( - "Overview - project-1 - Pagure", output_text - ) - self.assertIn("Added the README", output_text) - self.assertIn( - "/?next=http://localhost/fork/pingou/project-1", output_text - ) - - output = self.app.get("/fork/pingou/project-1/settings") - self.assertIn( - "This repository is on repoSpanner region default", - output.get_data(as_text=True), - ) - - # Verify that only pseudo repos exist, and no on-disk repos got created - repodirlist = os.listdir(os.path.join(self.path, "repos")) - self.assertEqual(repodirlist, ["pseudo"]) - - @print_repospanner_log - @patch.dict( - "pagure.config.config", - { - "ALLOW_HTTP_PULL_PUSH": True, - "ALLOW_HTTP_PUSH": True, - "HTTP_REPO_ACCESS_GITOLITE": False, - }, - ) - def test_http_pull(self): - """Test that the HTTP pull endpoint works for repoSpanner.""" - tests.create_projects(self.session) - tests.create_tokens(self.session) - tests.create_tokens_acl(self.session) - self.create_project_full("clonetest", {"create_readme": "y"}) - - # Verify the new project is indeed on repoSpanner - project = pagure.lib.query._get_project(self.session, "clonetest") - self.assertTrue(project.is_on_repospanner) - - # Unfortunately, actually testing a git clone would need the app to - # run on a TCP port, which the test environment doesn't do. - output = self.app.get( - "/clonetest.git/info/refs?service=git-upload-pack" - ) - self.assertEqual(output.status_code, 200) - self.assertEqual( - output.content_type, "application/x-git-upload-pack-advertisement" - ) - output_text = output.get_data(as_text=True) - self.assertIn("# service=git-upload-pack", output_text) - self.assertIn("agent=repoSpanner", output_text) - self.assertIn("symref=HEAD:refs/heads/master", output_text) - self.assertIn(" refs/heads/master\x00", output_text) - - output = self.app.post( - "/clonetest.git/git-upload-pack", - headers={"Content-Type": "application/x-git-upload-pack-request"}, - ) - self.assertEqual(output.status_code, 400) - output_text = output.get_data(as_text=True) - self.assertIn("Error processing your request", output_text) - - @print_repospanner_log - @patch.dict( - "pagure.config.config", - { - "ALLOW_HTTP_PULL_PUSH": True, - "ALLOW_HTTP_PUSH": True, - "HTTP_REPO_ACCESS_GITOLITE": False, - }, - ) - def test_http_push(self): - """Test that the HTTP push endpoint works for repoSpanner.""" - tests.create_projects(self.session) - tests.create_tokens(self.session) - tests.create_tokens_acl(self.session) - self.create_project_full("clonetest", {"create_readme": "y"}) - - # Verify the new project is indeed on repoSpanner - project = pagure.lib.query._get_project(self.session, "clonetest") - self.assertTrue(project.is_on_repospanner) - - # Unfortunately, actually testing a git clone would need the app to - # run on a TCP port, which the test environment doesn't do. - output = self.app.get( - "/clonetest.git/info/refs?service=git-receive-pack", - environ_overrides={"REMOTE_USER": "pingou"}, - ) - self.assertEqual(output.status_code, 200) - self.assertEqual( - output.content_type, "application/x-git-receive-pack-advertisement" - ) - output_text = output.get_data(as_text=True) - self.assertIn("# service=git-receive-pack", output_text) - self.assertIn("agent=repoSpanner", output_text) - self.assertIn("symref=HEAD:refs/heads/master", output_text) - self.assertIn(" refs/heads/master\x00", output_text) - - @print_repospanner_log - @patch("pagure.ui.app.admin_session_timedout") - def test_hooks(self, ast): - """Test hook setting and running works.""" - ast.return_value = False - pagure.cli.admin.session = self.session - - user = tests.FakeUser(username="foo") - with tests.user_set(self.app.application, user): - data = { - "name": "project-1", - "description": "Project #1", - "create_readme": "y", - "csrf_token": self.get_csrf(), - } - - output = self.app.post("/new/", data=data, follow_redirects=True) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - '
\nProject #1', output_text - ) - self.assertIn( - "Overview - project-1 - Pagure", output_text - ) - self.assertIn("Added the README", output_text) - - output = self.app.get("/project-1/settings") - self.assertIn( - "This repository is on repoSpanner region default", - output.get_data(as_text=True), - ) - - # Check file before the commit: - output = self.app.get("/project-1/raw/master/f/README.md") - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertEqual(output_text, "# project-1\n\nProject #1") - - with tests.user_set(self.app.application, user): - # Set editing Denied - self.set_auth_status(False) - - # Try to make an edit in the repo - data = { - "content": "foo\n bar\n baz", - "commit_title": "test commit", - "commit_message": "Online commit", - "email": "foo@bar.com", - "branch": "master", - "csrf_token": self.get_csrf(), - } - - output = self.app.post( - "/project-1/edit/master/f/README.md", - data=data, - follow_redirects=True, - ) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn("Remote hook declined the push: ", output_text) - self.assertIn( - "Denied push for ref 'refs/heads/master' for user 'foo'", - output_text, - ) - - # Check file after the commit: - output = self.app.get("/project-1/raw/master/f/README.md") - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertEqual(output_text, "# project-1\n\nProject #1") - - # Set editing Allowed - self.set_auth_status(True) - - # Try to make an edit in the repo - data = { - "content": "foo\n bar\n baz", - "commit_title": "test commit", - "commit_message": "Online commit", - "email": "foo@bar.com", - "branch": "master", - "csrf_token": self.get_csrf(), - } - - output = self.app.post( - "/project-1/edit/master/f/README.md", - data=data, - follow_redirects=True, - ) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - "Commits - project-1 - Pagure", output_text - ) - - # Check file after the commit: - output = self.app.get("/project-1/raw/master/f/README.md") - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertEqual(output_text, "foo\n bar\n baz") - - @print_repospanner_log - @patch.dict( - "pagure.config.config", - { - "PAGURE_ADMIN_USERS": ["pingou"], - "ALLOW_ADMIN_IGNORE_EXISTING_REPOS": True, - }, - ) - @patch("pagure.ui.app.admin_session_timedout") - def test_adopt_project(self, ast): - """Test adopting a project in repoSpanner works.""" - ast.return_value = False - - user = tests.FakeUser(username="foo") - with tests.user_set(self.app.application, user): - output = self.app.get("/new/") - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn("Create new Project", output_text) - - data = { - "name": "project-1", - "description": "Project #1", - "create_readme": "y", - "csrf_token": self.get_csrf(), - } - - output = self.app.post("/new/", data=data, follow_redirects=True) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - '
\nProject #1', output_text - ) - self.assertIn( - "Overview - project-1 - Pagure", output_text - ) - self.assertIn("Added the README", output_text) - - output = self.app.get("/project-1/settings") - self.assertIn( - "This repository is on repoSpanner region default", - output.get_data(as_text=True), - ) - - # Delete the project instance so that the actual repo remains - project = pagure.lib.query._get_project(self.session, "project-1") - self.session.delete(project) - self.session.commit() - shutil.rmtree(os.path.join(self.path, "repos", "pseudo")) - - user = tests.FakeUser(username="pingou") - with tests.user_set(self.app.application, user): - output = self.app.get("/project-1/") - self.assertEqual(output.status_code, 404) - - data = { - "name": "project-1", - "description": "Recreated project #1", - "create_readme": "false", - "csrf_token": self.get_csrf(), - } - output = self.app.post("/new/", data=data, follow_redirects=True) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - "Repo pagure/main/project-1 already exists", output_text - ) - - data["ignore_existing_repos"] = "y" - output = self.app.post("/new/", data=data, follow_redirects=True) - self.assertEqual(output.status_code, 200) - output_text = output.get_data(as_text=True) - self.assertIn( - '
\nRecreated project #1', - output_text, - ) - self.assertIn( - "Overview - project-1 - Pagure", output_text - ) - self.assertIn("Added the README", output_text) - - -if __name__ == "__main__": - unittest.main(verbosity=2)