From 431630139f6e0cc511043490054c684ec6a9d3f4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 1/6] Add the possibility to add/remove ssh keys on the authorized_keys file This is the base of the work we need to do to drop gitolite from pagure. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index 5817945..eaef5cd 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -1248,3 +1248,49 @@ def generate_archive( namespace=project.namespace, username=project.user.user if project.is_fork else None, ) + + +@conn.task(queue=pagure_config.get("AUTHORIZED_KEYS_QUEUE", None), bind=True) +@pagure_task +def add_key_to_authorized_keys(self, session, ssh_folder, username, sshkey): + """ Add the specified key to the the `authorized_keys` file of the + specified ssh folder. + """ + if not os.path.exists(ssh_folder): + _log.info("No folder '%s' found", ssh_folder) + return + + fullpath = os.path.join(ssh_folder, "authorized_keys") + _log.info("Add ssh key for user %s to %s", username, fullpath) + with open(fullpath, "a") as stream: + stream.write("\n") + stream.write( + "{0} {1}".format( + pagure_config["SSH_KEYS_OPTIONS"] % {"username": username}, + sshkey.strip(), + ) + ) + + +@conn.task(queue=pagure_config.get("AUTHORIZED_KEYS_QUEUE", None), bind=True) +@pagure_task +def remove_key_from_authorized_keys(self, session, ssh_folder, sshkey): + """ Remove the specified key from the the `authorized_keys` file of the + specified ssh folder. + """ + if not os.path.exists(ssh_folder): + _log.info("No folder '%s' found", ssh_folder) + return + + fullpath = os.path.join(ssh_folder, "authorized_keys") + _log.info("Removing ssh key in %s", fullpath) + output = [] + with open(fullpath, "r") as stream: + for row in stream.readlines(): + row = row.strip() + if sshkey in row: + continue + output.append(row) + + with open(fullpath, "w") as stream: + stream.write("\n".join(output)) diff --git a/pagure/ui/app.py b/pagure/ui/app.py index db166bb..362b1e8 100644 --- a/pagure/ui/app.py +++ b/pagure/ui/app.py @@ -21,6 +21,7 @@ from sqlalchemy.exc import SQLAlchemyError import pagure.exceptions import pagure.lib.git import pagure.lib.query +import pagure.lib.tasks import pagure.forms import pagure.ui.filters from pagure.config import config as pagure_config @@ -1201,6 +1202,16 @@ def add_user_sshkey(): user, pagure_config.get("GITOLITE_KEYDIR", None) ) pagure.lib.tasks.gitolite_post_compile_only.delay() + if ( + pagure_config.get("GIT_AUTH_BACKEND") + == "pagure_authorized_keys" + ): + _log.info("SSH FOLDER: %s", pagure_config.get("SSH_FOLDER")) + pagure.lib.tasks.add_key_to_authorized_keys.delay( + ssh_folder=pagure_config.get("SSH_FOLDER"), + username=flask.g.fas_user.username, + sshkey=form.ssh_key.data, + ) flask.flash(msg) return flask.redirect( flask.url_for("ui_ns.user_settings") + "#nav-ssh-tab" @@ -1234,8 +1245,10 @@ def remove_user_sshkey(keyid): if form.validate_on_submit(): user = _get_user(username=flask.g.fas_user.username) found = False + sshkey = None for key in user.sshkeys: if key.id == keyid: + sshkey = key.public_ssh_key flask.g.session.delete(key) found = True break @@ -1252,6 +1265,14 @@ def remove_user_sshkey(keyid): user, pagure_config.get("GITOLITE_KEYDIR", None) ) pagure.lib.tasks.gitolite_post_compile_only.delay() + if ( + pagure_config.get("GIT_AUTH_BACKEND") + == "pagure_authorized_keys" + ): + _log.info("SSH FOLDER: %s", pagure_config.get("SSH_FOLDER")) + pagure.lib.tasks.remove_key_from_authorized_keys.delay( + ssh_folder=pagure_config.get("SSH_FOLDER"), sshkey=sshkey + ) flask.flash("SSH key removed") except SQLAlchemyError as err: # pragma: no cover flask.g.session.rollback() diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index aa1201b..93f5c65 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -1833,8 +1833,10 @@ def remove_deploykey(repo, keyid, username=None, namespace=None): form = pagure.forms.ConfirmationForm() if form.validate_on_submit(): found = False + sshkey = None for key in repo.deploykeys: if key.id == keyid: + sshkey = key.public_ssh_key flask.g.session.delete(key) found = True break @@ -1857,6 +1859,14 @@ def remove_deploykey(repo, keyid, username=None, namespace=None): repo, pagure_config.get("GITOLITE_KEYDIR", None) ) pagure.lib.tasks.gitolite_post_compile_only.delay() + if ( + pagure_config.get("GIT_AUTH_BACKEND") + == "pagure_authorized_keys" + ): + _log.info("SSH FOLDER: %s", pagure_config.get("SSH_FOLDER")) + pagure.lib.tasks.remove_key_from_authorized_keys.delay( + ssh_folder=pagure_config.get("SSH_FOLDER"), sshkey=sshkey + ) flask.flash("Deploy key removed") except SQLAlchemyError as err: # pragma: no cover flask.g.session.rollback() @@ -1981,6 +1991,16 @@ def add_deploykey(repo, username=None, namespace=None): repo, pagure_config.get("GITOLITE_KEYDIR", None) ) pagure.lib.tasks.gitolite_post_compile_only.delay() + if ( + pagure_config.get("GIT_AUTH_BACKEND") + == "pagure_authorized_keys" + ): + _log.info("SSH FOLDER: %s", pagure_config.get("SSH_FOLDER")) + pagure.lib.tasks.add_key_to_authorized_keys.delay( + ssh_folder=pagure_config.get("SSH_FOLDER"), + username=flask.g.fas_user.username, + sshkey=form.ssh_key.data, + ) flask.flash(msg) return flask.redirect( flask.url_for( diff --git a/setup.py b/setup.py index c7d9aa6..e9c3646 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ setup( gitolite2 = pagure.lib.git_auth:Gitolite2Auth gitolite3 = pagure.lib.git_auth:Gitolite3Auth pagure = pagure.lib.git_auth:PagureGitAuth + pagure_authorized_keys = pagure.lib.git_auth:PagureGitAuth """, classifiers=[ "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", From 64649607862c0b3ad6e4e2e072ab4883538fbd16 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 2/6] Don't set the log level on the console handler by default We're setting the logging level on loggers instead. Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/default_config.py b/pagure/default_config.py index 282ff87..710581d 100644 --- a/pagure/default_config.py +++ b/pagure/default_config.py @@ -454,7 +454,6 @@ LOGGING = { "filters": {"myfilter": {"()": ContextInjector}}, "handlers": { "console": { - "level": "INFO", "formatter": "standard", "class": "logging.StreamHandler", "stream": "ext://sys.stdout", From 410e1c214e0fd8e0ad0d07161c6583ceb16807ca Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 3/6] Increase the logging when calling git for http-based operations Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/ui/clone.py b/pagure/ui/clone.py index e04e137..a1eba21 100644 --- a/pagure/ui/clone.py +++ b/pagure/ui/clone.py @@ -39,6 +39,7 @@ def proxy_raw_git(): This should get called after it is determined the requested project is not on repoSpanner. """ + _log.debug("Raw git clone proxy started") # We are going to shell out to gitolite-shell. Prepare the env it needs. gitenv = { "PATH": os.environ["PATH"], @@ -93,8 +94,10 @@ def proxy_raw_git(): raise ValueError("Value for key %s unknown" % key) if gitolite: + _log.debug("Running git via: %s", gitolite) cmd = [gitolite] else: + _log.debug("Running git via git directly") cmd = ["/usr/bin/git", "http-backend"] # Note: using a temporary files to buffer the input contents @@ -111,6 +114,7 @@ def proxy_raw_git(): infile.write(block) infile.seek(0) + _log.debug("Calling: %s", cmd) proc = subprocess.Popen( cmd, stdin=infile, stdout=subprocess.PIPE, stderr=None, env=gitenv ) From 80f1594d58df46243a09bd60526fdf32ea849d12 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 4/6] Change the formatting of the bashrc file Have one service per line, this makes it easier to read as well as clearer when changing it. Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/ansible/roles/pagure-dev/files/bashrc b/dev/ansible/roles/pagure-dev/files/bashrc index 8eede33..634457d 100644 --- a/dev/ansible/roles/pagure-dev/files/bashrc +++ b/dev/ansible/roles/pagure-dev/files/bashrc @@ -21,25 +21,41 @@ echo "Reminder of the commands: export PAGURE_CONFIG=~/pagure.cfg pstart (){ - sudo systemctl start pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service pagure_worker.service + sudo systemctl start pagure.service \ + pagure-docs.service \ + pagure_ci.service \ + pagure_ev.service \ + pagure_webhook.service \ + pagure_worker.service echo 'The application is running on http://localhost:5000/' } pstop (){ - sudo systemctl stop pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service pagure_worker.service + sudo systemctl stop pagure.service \ + pagure-docs.service \ + pagure_ci.service \ + pagure_ev.service \ + pagure_webhook.service \ + pagure_worker.service } prestart (){ - sudo systemctl restart pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service pagure_worker.service + sudo systemctl restart pagure.service \ + pagure-docs.service \ + pagure_ci.service \ + pagure_ev.service \ + pagure_webhook.service \ + pagure_worker.service echo 'The application is running on http://localhost:5000/' } pstatus (){ - sudo systemctl status pagure.service pagure-docs.service pagure_ci.service\ - pagure_ev.service pagure_webhook.service pagure_worker.service + sudo systemctl status pagure.service \ + pagure-docs.service \ + pagure_ci.service \ + pagure_ev.service \ + pagure_webhook.service \ + pagure_worker.service } alias plog="sudo journalctl -lu pagure.service" From 67f8972bcdd300656896cd11163a14d1b4f697f5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 5/6] Add a new pagure_authorized_keys service This is the service that will managed the authorized_keys file when using the ``pagure_authorized_keys`` auth backend. Enable this service in the vagrant-based dev environment Include a production-ready version of this service in the `files/` folder to be included in the releases. Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/ansible/roles/pagure-dev/files/bashrc b/dev/ansible/roles/pagure-dev/files/bashrc index 634457d..a14f760 100644 --- a/dev/ansible/roles/pagure-dev/files/bashrc +++ b/dev/ansible/roles/pagure-dev/files/bashrc @@ -26,7 +26,8 @@ pstart (){ pagure_ci.service \ pagure_ev.service \ pagure_webhook.service \ - pagure_worker.service + pagure_worker.service \ + pagure_authorized_keys_worker.service echo 'The application is running on http://localhost:5000/' } @@ -36,7 +37,8 @@ pstop (){ pagure_ci.service \ pagure_ev.service \ pagure_webhook.service \ - pagure_worker.service + pagure_worker.service \ + pagure_authorized_keys_worker.service } prestart (){ @@ -45,7 +47,8 @@ prestart (){ pagure_ci.service \ pagure_ev.service \ pagure_webhook.service \ - pagure_worker.service + pagure_worker.service \ + pagure_authorized_keys_worker.service echo 'The application is running on http://localhost:5000/' } @@ -55,7 +58,8 @@ pstatus (){ pagure_ci.service \ pagure_ev.service \ pagure_webhook.service \ - pagure_worker.service + pagure_worker.service \ + pagure_authorized_keys_worker.service } alias plog="sudo journalctl -lu pagure.service" diff --git a/dev/ansible/roles/pagure-dev/files/pagure_authorized_keys_worker.service b/dev/ansible/roles/pagure-dev/files/pagure_authorized_keys_worker.service new file mode 100644 index 0000000..296db92 --- /dev/null +++ b/dev/ansible/roles/pagure-dev/files/pagure_authorized_keys_worker.service @@ -0,0 +1,15 @@ +[Unit] +Description=Pagure worker for managing the authorized_keys file +After=redis.target +Documentation=https://pagure.io/pagure + +[Service] +ExecStart=celery worker -A pagure.lib.tasks --loglevel=info -c 1 -Q authorized_keys_queue +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +Type=simple +Restart=on-failure +User=git +Group=git + +[Install] +WantedBy=default.target diff --git a/dev/ansible/roles/pagure-dev/tasks/main.yml b/dev/ansible/roles/pagure-dev/tasks/main.yml index 103d73e..5b18c81 100644 --- a/dev/ansible/roles/pagure-dev/tasks/main.yml +++ b/dev/ansible/roles/pagure-dev/tasks/main.yml @@ -198,6 +198,7 @@ - pagure_ev.service - pagure_webhook.service - pagure_worker.service + - pagure_authorized_keys_worker.service - name: let paguremirroring read the pagure config command: /usr/bin/setfacl -m user:paguremirroring:rx /etc/pagure/pagure.cfg @@ -238,3 +239,4 @@ - pagure_ev - pagure_webhook - pagure_worker + - pagure_authorized_keys_worker diff --git a/files/pagure_authorized_keys_worker.service b/files/pagure_authorized_keys_worker.service new file mode 100644 index 0000000..f5f6a05 --- /dev/null +++ b/files/pagure_authorized_keys_worker.service @@ -0,0 +1,15 @@ +[Unit] +Description=Pagure worker for managing the authorized_keys file +After=redis.target +Documentation=https://pagure.io/pagure + +Service] +ExecStart=/usr/bin/celery worker -A pagure.lib.tasks --loglevel=info -c 1 -Q authorized_keys_queue +Environment="PAGURE_CONFIG=/etc/pagure/pagure.cfg" +Type=simple +User=git +Group=git +Restart=on-failure + +[Install] +WantedBy=default.target From 15710f042b34265a543ee0cdeae78b2e39bbe12b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Apr 30 2020 12:09:40 +0000 Subject: [PATCH 6/6] Adjust the pagure config in vagrant to deploy with the new backend by default Signed-off-by: Pierre-Yves Chibon --- diff --git a/dev/ansible/roles/pagure-dev/files/pagure.cfg b/dev/ansible/roles/pagure-dev/files/pagure.cfg index 3ca257d..de2f409 100644 --- a/dev/ansible/roles/pagure-dev/files/pagure.cfg +++ b/dev/ansible/roles/pagure-dev/files/pagure.cfg @@ -55,6 +55,19 @@ REMOTE_GIT_FOLDER = os.path.join(STORAGE_ROOT, 'remotes') ### Whether to enable scanning for viruses in attachments VIRUS_SCAN_ATTACHMENTS = False +SSH_FOLDER = "/srv/git/.ssh/" + +GIT_AUTH_BACKEND = "pagure_authorized_keys" + +SSH_KEYS_OPTIONS = ( + 'restrict,command="/usr/bin/python3 /srv/pagure/files/aclchecker.py %(username)s"' +) + +SSH_COMMAND_NON_REPOSPANNER = ([ + "/usr/bin/%(cmd)s", + "/srv/git/repositories/%(reponame)s", +], {"GL_USER": "%(username)s"}) + # SSH Information