From 0811869364a25ccefc427479162bfad38aac36bd Mon Sep 17 00:00:00 2001 From: Stephen Coady Date: Mar 29 2021 13:49:56 +0000 Subject: [PATCH 1/2] Merge branch 'develop' into production Signed-off-by: Stephen Coady --- diff --git a/.gitignore b/.gitignore index b8bb54d..f4d8b50 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ build/ dist/ .coverage +coverage.xml +client_secrets.json alembic.ini .vagrant/ .tox/ diff --git a/README.md b/README.md index 325ada6..35c96d4 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ development environment to test your changes. This is simplified by using [Vagrant](https://www.vagrantup.com/ "Vagrant by Hashicorp"), a powerful and useful tool for creating development environments on your workstation. + ### Using Vagrant You can quickly start hacking on the Fedora Elections web application using the @@ -46,6 +47,7 @@ vagrant ssh Once that is running, go to [localhost:5005](http://localhost:5005/) in your browser to see your running Fedora Elections test instance. + ### A note about fonts Fedora Elections uses web fonts hosted in Fedora's infrastructure that might @@ -104,30 +106,40 @@ You can obtain the code via: git clone https://pagure.io/elections.git ``` -### Configure the application +### Install pip requirements w/ tox for testing -An example configuration file is provided [here](https://pagure.io/elections/blob/master/f/files/fedora-elections.cfg "files/fedora-elections.cfg"). +Set up venv (replacing `` and ``): -### Create database +``` +pip install --user virtualenv tox +mkvirtualenv / +. / +``` -Run: +Install requirements: ``` -python createdb.py +pip install -r requirements.txt ``` +### Configure the application + +An example configuration file is provided [here](https://pagure.io/elections/blob/master/f/files/fedora-elections.cfg "files/fedora-elections.cfg"). + ### Register the application using openid-connect -Run: +From root of project, run: ``` -oidc-register https://iddev.fedorainfracloud.org/ http://localhost:5005/oidc_callback +oidc-register https://iddev.fedorainfracloud.org/ http://localhost:5005 ``` -Copy the corresponding ``client_secrets.json`` in the sources: +### Create database + +Run: ``` -cp client_secrets.json fedora_elections/client_secrets.json +python createdb.py ``` ### Create a local configuration file @@ -192,6 +204,18 @@ sudo systemctl restart httpd ``` +### Running tests +fedora-elections uses `tox` to simplify testing, and support testing across multiple environments. + +Refer to earlier in this section on "How to launch Fedora Elections" in this document to get set up with `tox` + +To run tests, simply run: + +``` +tox +``` + + ## How to contribute As mentioned earlier, this project is primarily hosted on [Pagure](https://pagure.io/elections "Fedora Infrastructure Elections application"). diff --git a/fedora_elections/__init__.py b/fedora_elections/__init__.py index 20e7865..2a34232 100644 --- a/fedora_elections/__init__.py +++ b/fedora_elections/__init__.py @@ -43,6 +43,7 @@ import flask # noqa import munch # noqa import six # noqa +from fasjson_client import Client from fedora.client import AuthError, AppError # noqa from fedora.client.fas2 import AccountSystem # noqa from flask_oidc import OpenIDConnect # noqa @@ -64,13 +65,18 @@ LOG = APP.logger APP.wsgi_app = fedora_elections.proxy.ReverseProxied(APP.wsgi_app) -# FAS for usernames. -FAS2 = AccountSystem( - APP.config['FAS_BASE_URL'], - username=APP.config['FAS_USERNAME'], - password=APP.config['FAS_PASSWORD'], - insecure=not APP.config['FAS_CHECK_CERT'] -) +if APP.config.get('FASJSON'): + ACCOUNTS = Client( + url=APP.config['FAS_BASE_URL'] + ) +else: + # FAS for usernames. + ACCOUNTS = AccountSystem( + APP.config['FAS_BASE_URL'], + username=APP.config['FAS_USERNAME'], + password=APP.config['FAS_PASSWORD'], + insecure=not APP.config['FAS_CHECK_CERT'] + ) # modular imports @@ -210,8 +216,8 @@ def set_session(): # pragma: no-cover 'email': OIDC.user_getfield('email') or '', 'timezone': OIDC.user_getfield('zoneinfo'), 'cla_done': - 'http://admin.fedoraproject.org/accounts/cla/done' - in (OIDC.user_getfield('cla') or []), + 'FPCA' + in (OIDC.user_getfield('agreements') or []), }) flask.g.fas_user = flask.session.fas_user else: diff --git a/fedora_elections/admin.py b/fedora_elections/admin.py index beab90e..3c013fd 100644 --- a/fedora_elections/admin.py +++ b/fedora_elections/admin.py @@ -31,13 +31,21 @@ from functools import wraps import flask from sqlalchemy.exc import SQLAlchemyError from fedora.client import AuthError +from fedora_elections_messages import ( + NewElectionV1, + EditElectionV1, + NewCandidateV1, + EditCandidateV1, + DeleteCandidateV1, +) from fedora_elections import fedmsgshim from fedora_elections import forms from fedora_elections import models from fedora_elections import ( - APP, SESSION, FAS2, is_authenticated, is_admin + APP, SESSION, ACCOUNTS, is_authenticated, is_admin ) +from fasjson_client.errors import APIError def election_admin_required(f): @@ -108,13 +116,11 @@ def admin_new_election(): SESSION.commit() - fedmsgshim.publish( - topic="election.new", - msg=dict( + fedmsgshim.publish(NewElectionV1(body=dict( agent=flask.g.fas_user.username, election=election.to_json(), - ) - ) + ) + )) flask.flash('Election "%s" added' % election.alias) return flask.redirect(flask.url_for( @@ -196,13 +202,11 @@ def admin_view_election(election_alias): SESSION.delete(admingrp) SESSION.commit() - fedmsgshim.publish( - topic="election.edit", - msg=dict( + fedmsgshim.publish(EditElectionV1(body=dict( agent=flask.g.fas_user.username, election=election.to_json(), ) - ) + )) flask.flash('Election "%s" saved' % election.alias) return flask.redirect(flask.url_for( 'admin_view_election', election_alias=election.alias)) @@ -230,9 +234,14 @@ def admin_add_candidate(election_alias): fas_name = None if election.candidates_are_fasusers: # pragma: no cover try: - fas_name = FAS2.person_by_username( - form.name.data)['human_name'] - except (KeyError, AuthError): + if APP.config.get('FASJSON'): + user = ACCOUNTS.get_user( + username=form.name.data).result + fas_name = f"{user['givenname']} {user['surname']}" + else: + fas_name = ACCOUNTS.person_by_username( + form.name.data)['human_name'] + except (KeyError, AuthError, APIError): flask.flash( 'User `%s` does not have a FAS account.' % form.name.data, 'error') @@ -251,14 +260,12 @@ def admin_add_candidate(election_alias): SESSION.add(candidate) SESSION.commit() flask.flash('Candidate "%s" saved' % candidate.name) - fedmsgshim.publish( - topic="candidate.new", - msg=dict( + fedmsgshim.publish(NewCandidateV1(body=dict( agent=flask.g.fas_user.username, election=candidate.election.to_json(), candidate=candidate.to_json(), ) - ) + )) return flask.redirect(flask.url_for( 'admin_view_election', election_alias=election.alias)) @@ -286,9 +293,14 @@ def admin_add_multi_candidate(election_alias): fas_name = None if election.candidates_are_fasusers: # pragma: no cover try: - fas_name = FAS2.person_by_username( - candidate[0])['human_name'] - except (KeyError, AuthError): + if APP.config.get('FASJSON'): + user = ACCOUNTS.get_user( + username=candidate[0]).result + fas_name = f"{user['givenname']} {user['surname']}" + else: + fas_name = ACCOUNTS.person_by_username( + candidate[0])['human_name'] + except (KeyError, AuthError, APIError): SESSION.rollback() flask.flash( 'User `%s` does not have a FAS account.' @@ -317,14 +329,13 @@ def admin_add_multi_candidate(election_alias): candidates_name.append(cand.name) else: flask.flash("There was an issue!") - fedmsgshim.publish( - topic="candidate.new", - msg=dict( + continue + fedmsgshim.publish(NewCandidateV1(body=dict( agent=flask.g.fas_user.username, election=cand.election.to_json(), candidate=cand.to_json(), ) - ) + )) SESSION.commit() flask.flash('Added %s candidates' % len(candidates_name)) @@ -356,9 +367,14 @@ def admin_edit_candidate(election_alias, candidate_id): if election.candidates_are_fasusers: # pragma: no cover try: - candidate.fas_name = FAS2.person_by_username( - candidate.name)['human_name'] - except (KeyError, AuthError): + if APP.config.get('FASJSON'): + user = ACCOUNTS.get_user( + username=candidate.name).result + candidate.fas_name = f"{user['givenname']} {user['surname']}" + else: + candidate.fas_name = ACCOUNTS.person_by_username( + candidate.name)['human_name'] + except (KeyError, AuthError, APIError): SESSION.rollback() flask.flash( 'User `%s` does not have a FAS account.' @@ -370,14 +386,12 @@ def admin_edit_candidate(election_alias, candidate_id): SESSION.commit() flask.flash('Candidate "%s" saved' % candidate.name) - fedmsgshim.publish( - topic="candidate.edit", - msg=dict( + fedmsgshim.publish(EditCandidateV1(body=dict( agent=flask.g.fas_user.username, election=candidate.election.to_json(), candidate=candidate.to_json(), ) - ) + )) return flask.redirect(flask.url_for( 'admin_view_election', election_alias=election.alias)) @@ -407,14 +421,12 @@ def admin_delete_candidate(election_alias, candidate_id): SESSION.delete(candidate) SESSION.commit() flask.flash('Candidate "%s" deleted' % candidate_name) - fedmsgshim.publish( - topic="candidate.delete", - msg=dict( + fedmsgshim.publish(DeleteCandidateV1(body=dict( agent=flask.g.fas_user.username, election=candidate.election.to_json(), candidate=candidate.to_json(), ) - ) + )) except SQLAlchemyError as err: SESSION.rollback() APP.logger.debug('Could not delete candidate') diff --git a/fedora_elections/default_config.py b/fedora_elections/default_config.py index 42c6d44..73b86c8 100644 --- a/fedora_elections/default_config.py +++ b/fedora_elections/default_config.py @@ -19,6 +19,8 @@ DB_URL = 'sqlite:////var/tmp/elections_dev.sqlite' # You will want to change this for your install SECRET_KEY = 'change me' +FASJSON = False + FAS_BASE_URL = 'https://admin.stg.fedoraproject.org/accounts/' FAS_USERNAME = '' FAS_PASSWORD = '' @@ -27,7 +29,7 @@ FAS_CHECK_CERT = False OIDC_CLIENT_SECRETS = os.path.join(os.path.dirname( os.path.abspath(__file__)), '..', 'client_secrets.json') -OIDC_SCOPES = ['openid', 'email', 'profile', 'fedora'] +OIDC_SCOPES = ['openid', 'email', 'profile', 'https://id.fedoraproject.org/scope/groups', 'https://id.fedoraproject.org/scope/agreements'] OIDC_OPENID_REALM = 'http://localhost:5005/oidc_callback' LOGGING = { diff --git a/fedora_elections/fedmsgshim.py b/fedora_elections/fedmsgshim.py index ce08619..8d8eed0 100644 --- a/fedora_elections/fedmsgshim.py +++ b/fedora_elections/fedmsgshim.py @@ -15,13 +15,9 @@ from fedora_messaging.exceptions import PublishReturned, ConnectionException _log = logging.getLogger(__name__) -def publish(topic, msg): # pragma: no cover - _log.debug('Publishing a message for %r: %s', topic, msg) +def publish(message): # pragma: no cover + _log.debug('Publishing a message for %r: %s', message.topic, message.body) try: - message = fedora_messaging.api.Message( - topic='fedora_elections.%s' % topic, - body=msg - ) fedora_messaging.api.publish(message) _log.debug("Sent to fedora_messaging") except PublishReturned as e: diff --git a/fedora_elections/forms.py b/fedora_elections/forms.py index b88edaf..0c927cd 100644 --- a/fedora_elections/forms.py +++ b/fedora_elections/forms.py @@ -10,8 +10,9 @@ except ImportError: from fedora.client import AuthError -from fedora_elections import SESSION, FAS2, APP +from fedora_elections import SESSION, ACCOUNTS, APP from fedora_elections.models import Election +from fasjson_client.errors import APIError class ElectionForm(FlaskForm): @@ -155,9 +156,14 @@ def get_simple_voting_form(candidates, fasusers): if fasusers: # pragma: no cover # We can't cover FAS integration try: - title = \ - FAS2.person_by_username(candidate.name)['human_name'] - except (KeyError, AuthError) as err: + if APP.config.get('FASJSON'): + user = ACCOUNTS.get_user( + username=candidate.name).result + title = f"{user['givenname']} {user['surname']}" + else: + title = ACCOUNTS.person_by_username( + candidate.name)['human_name'] + except (KeyError, AuthError, APIError) as err: APP.logger.debug(err) if candidate.url: title = '%s [Info]' % (title, candidate.url) diff --git a/fedora_elections/models.py b/fedora_elections/models.py index b447bb6..703cc3d 100644 --- a/fedora_elections/models.py +++ b/fedora_elections/models.py @@ -267,6 +267,7 @@ class Candidate(BASE): ''' Return a json representation of this object. ''' return dict( name=self.name, + fas_name=self.fas_name, url=self.url, ) diff --git a/files/fedora-elections.cfg b/files/fedora-elections.cfg index 0ce7c61..37e76e5 100644 --- a/files/fedora-elections.cfg +++ b/files/fedora-elections.cfg @@ -15,6 +15,11 @@ DB_URL = 'sqlite:////var/tmp/elections_dev.sqlite' ## application, including all elections past, present and future FEDORA_ELECTIONS_ADMIN_GROUP = 'elections' +# Elections directly connects to the accounts backend to get +# details of nominees when adding them to an election. +# if FASJSON is false, elections will connect to FAS2. if FASJSON is +# True, elections will connect to FASJSON +FASJSON = False ## Fedora-elections can integrate with FAS to retrieve information about the ## candidates, the following configuration keys are required for this diff --git a/requirements.txt b/requirements.txt index 981ee1c..2adb277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,5 @@ six gunicorn psycopg2 rsa==4.0 # last version that supports python2 +fasjson_client +fedora_elections_messages diff --git a/runserver.py b/runserver.py index 9b516c1..54cbbea 100644 --- a/runserver.py +++ b/runserver.py @@ -26,6 +26,12 @@ parser.add_argument( '--port', '-p', default=5005, help='Port for the flask application.') parser.add_argument( + '--cert', '-s', default=None, + help='Filename of SSL cert for the flask application.') +parser.add_argument( + '--key', '-k', default=None, + help='Filename of the SSL key for the flask application.') +parser.add_argument( '--host', default="127.0.0.1", help='Hostname to listen on. When set to 0.0.0.0 the server is available \ externally. Defaults to 127.0.0.1 making the it only visable on localhost') @@ -47,4 +53,7 @@ if args.config: os.environ['FEDORA_ELECTIONS_CONFIG'] = config APP.debug = True -APP.run(host=args.host, port=int(args.port)) +if args.cert and args.key: + APP.run(host=args.host, port=int(args.port), ssl_context=(args.cert, args.key)) +else: + APP.run(host=args.host, port=int(args.port)) diff --git a/setup.py b/setup.py index 8836c03..c9e8a12 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( install_requires=[ 'Flask', 'SQLAlchemy>=0.7', 'python-fedora', 'kitchen', 'python-openid', 'python-openid-teams', 'python-openid-cla', - 'Flask-wtf', 'wtforms', + 'Flask-wtf', 'wtforms', 'fedora-elections-messages', ], test_suite="tests", ) diff --git a/tests/test_candidate.py b/tests/test_candidate.py index aab8751..1b07da5 100644 --- a/tests/test_candidate.py +++ b/tests/test_candidate.py @@ -218,6 +218,7 @@ class Candidatetests(Modeltests): { 'name': 'Ralph', 'url': 'https://fedoraproject.org/wiki/User:Ralph', + 'fas_name': None, } ) @@ -227,6 +228,7 @@ class Candidatetests(Modeltests): { 'name': 'Kevin', 'url': 'https://fedoraproject.org/wiki/User:Kevin', + 'fas_name': None, } ) diff --git a/tests/test_flask_admin.py b/tests/test_flask_admin.py index 468d24d..4cc0feb 100644 --- a/tests/test_flask_admin.py +++ b/tests/test_flask_admin.py @@ -33,6 +33,10 @@ from datetime import timedelta import flask from mock import patch, MagicMock +from fedora_messaging.testing import mock_sends +from fedora_elections_messages import ( + NewElectionV1, EditElectionV1, NewCandidateV1, EditCandidateV1, DeleteCandidateV1, +) sys.path.insert(0, os.path.join(os.path.dirname( os.path.abspath(__file__)), '..')) @@ -318,8 +322,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/new', data=data, follow_redirects=True) + with mock_sends(NewElectionV1): + output = self.app.post( + '/admin/new', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -357,8 +362,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/new', data=data, follow_redirects=True) + with mock_sends(NewElectionV1): + output = self.app.post( + '/admin/new', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -502,8 +508,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election/', data=data, follow_redirects=True) + with mock_sends(EditElectionV1): + output = self.app.post( + '/admin/test_election/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -580,8 +587,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election2/', data=data, follow_redirects=True) + with mock_sends(EditElectionV1): + output = self.app.post( + '/admin/test_election2/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -619,8 +627,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election2/', data=data, follow_redirects=True) + with mock_sends(EditElectionV1): + output = self.app.post( + '/admin/test_election2/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -705,8 +714,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election3/', data=data, follow_redirects=True) + with mock_sends(EditElectionV1): + output = self.app.post( + '/admin/test_election3/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -734,8 +744,9 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election3/', data=data, follow_redirects=True) + with mock_sends(EditElectionV1): + output = self.app.post( + '/admin/test_election3/', data=data, follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -829,9 +840,10 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election/candidates/new', data=data, - follow_redirects=True) + with mock_sends(NewCandidateV1): + output = self.app.post( + '/admin/test_election/candidates/new', data=data, + follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -919,9 +931,10 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election/candidates/new/multi', data=data, - follow_redirects=True) + with mock_sends(NewCandidateV1, NewCandidateV1, NewCandidateV1): + output = self.app.post( + '/admin/test_election/candidates/new/multi', data=data, + follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -1029,9 +1042,10 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election/candidates/1/edit', data=data, - follow_redirects=True) + with mock_sends(EditCandidateV1): + output = self.app.post( + '/admin/test_election/candidates/1/edit', data=data, + follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( @@ -1121,9 +1135,10 @@ class FlaskAdmintests(ModelFlasktests): 'csrf_token': csrf_token, } - output = self.app.post( - '/admin/test_election4/candidates/10/delete', data=data, - follow_redirects=True) + with mock_sends(DeleteCandidateV1): + output = self.app.post( + '/admin/test_election4/candidates/10/delete', data=data, + follow_redirects=True) self.assertEqual(output.status_code, 200) output_text = output.get_data(as_text=True) self.assertTrue( diff --git a/tox.ini b/tox.ini index 4d3302a..3728341 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,diff-cover +envlist = py36,py37,py38,diff-cover skipsdist = True [testenv] From 781267e677c438264b4eb520d7a06068c64d4abc Mon Sep 17 00:00:00 2001 From: Stephen Coady Date: Mar 29 2021 13:54:53 +0000 Subject: [PATCH 2/2] prep release 2.10 Signed-off-by: Stephen Coady --- diff --git a/fedora_elections/__init__.py b/fedora_elections/__init__.py index 2a34232..ee0baa5 100644 --- a/fedora_elections/__init__.py +++ b/fedora_elections/__init__.py @@ -25,7 +25,7 @@ # from __future__ import unicode_literals, absolute_import -__version__ = '2.9' +__version__ = '2.10' import logging # noqa import os # noqa diff --git a/files/fedora-elections.spec b/files/fedora-elections.spec index f84d23e..31dc0dd 100644 --- a/files/fedora-elections.spec +++ b/files/fedora-elections.spec @@ -1,7 +1,7 @@ %define modname fedora_elections Name: fedora-elections -Version: 2.9 +Version: 2.10 Release: 1%{?dist} Summary: Fedora elections application @@ -112,6 +112,13 @@ install -m 644 files/update_1_to_2.sql \ %changelog +* Mon Mar 29 2021 Stephen Coady 2.10-1 +- Update to 2.10 +- add fasjson support +- general bugfixes +- make use of fedora messaging schemas +- use new oidc scopes + * Mon Nov 18 2019 Ben Cotton 2.9-1 - Update to 2.9 (2.8 existed, in a sense) - Open "more info" links in a new window