From 3359be026581248705f7082012291be0c1fd28dd Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jul 29 2017 16:24:16 +0000 Subject: [PATCH 1/4] Use Honcho to start the services in the VM in dev mode --- diff --git a/ansible/files/motd b/ansible/files/motd index fdd814a..b5fd69d 100644 --- a/ansible/files/motd +++ b/ansible/files/motd @@ -7,11 +7,11 @@ Here are some tips: * You can type `workon hubs` to enter a configured Python virtualenv -* Run `hup` to start the flask webserver +* Run `honcho start` to start the webserver and the backend services. * Run `hreset` to delete the database and repopulate it -Once you run `hup` you can navigate to http://localhost:5000/ +Once you run `honcho start` you can navigate to http://localhost:5000/ in your browser. Happy hacking! diff --git a/ansible/roles/hubs/tasks/dev.yml b/ansible/roles/hubs/tasks/dev.yml new file mode 100644 index 0000000..9b3133c --- /dev/null +++ b/ansible/roles/hubs/tasks/dev.yml @@ -0,0 +1,23 @@ +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - python-sphinx + - python2-devel + - python3-devel + - python3-honcho + - python3-tox + - redhat-rpm-config + - sqlite-devel + +- name: Install a custom bashrc + template: src=bashrc dest=/home/{{ main_user }}/.bashrc + +- name: Install Honcho's env file + template: src=honcho-env dest={{ hubs_base_dir }}/.env + +- name: Install Honcho's procfile + template: src=honcho-procfile dest={{ hubs_base_dir }}/Procfile diff --git a/ansible/roles/hubs/tasks/main.yml b/ansible/roles/hubs/tasks/main.yml index be0c8c9..2aa310a 100644 --- a/ansible/roles/hubs/tasks/main.yml +++ b/ansible/roles/hubs/tasks/main.yml @@ -14,21 +14,6 @@ - python-virtualenv - python3-flask-oidc -- name: Install Fedora Hubs development packages - dnf: name={{ item }} state=present - with_items: - - gcc - - gcc-c++ - - libffi-devel - - openssl-devel - - python-sphinx - - python2-devel - - python3-devel - - python3-tox - - redhat-rpm-config - - sqlite-devel - when: hubs_dev_mode - - name: Install the distribution versions of requirements.txt dnf: name={{ item }} state=present with_items: @@ -56,12 +41,6 @@ - python-retask -# Add various helpful configuration files -- name: Install a custom bashrc - template: src=bashrc dest=/home/{{ main_user }}/.bashrc - when: hubs_dev_mode - - # Create directory structure - name: Create the directory structure @@ -169,33 +148,16 @@ - name: Disable the system-wide fedmsg-hub service: name=fedmsg-hub state=stopped enabled=no -- name: Install the service files - template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service - with_items: - - hubs-triage@ - - hubs-worker@ - - hubs-sse - - hubs-fedmsg-hub - register: service_installed - -- name: reload systemd - command: systemctl daemon-reload - when: service_installed|changed - -- name: Start and enable the services +- name: Start and enable the common services service: name={{ item }} state=started enabled=yes with_items: - redis - - hubs-triage@1 - - hubs-triage@2 - - hubs-worker@1 - - hubs-worker@2 - - hubs-sse - - hubs-fedmsg-hub -# Webserver -- include: webserver.yml +# Include mode-specific tasks + +- include: dev.yml + when: hubs_dev_mode + +- include: prod.yml when: not hubs_dev_mode diff --git a/ansible/roles/hubs/tasks/prod.yml b/ansible/roles/hubs/tasks/prod.yml new file mode 100644 index 0000000..0f8bb3d --- /dev/null +++ b/ansible/roles/hubs/tasks/prod.yml @@ -0,0 +1,26 @@ +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-triage@ + - hubs-worker@ + - hubs-sse + - hubs-fedmsg-hub + register: service_installed + +- name: reload systemd + command: systemctl daemon-reload + when: service_installed|changed + +- name: Start and enable the services in prod mode + service: name={{ item }} state=started enabled=yes + with_items: + - hubs-triage@1 + - hubs-triage@2 + - hubs-worker@1 + - hubs-worker@2 + - hubs-sse + - hubs-fedmsg-hub + +- include: webserver.yml diff --git a/ansible/roles/hubs/templates/bashrc b/ansible/roles/hubs/templates/bashrc index 8902729..f7004fa 100644 --- a/ansible/roles/hubs/templates/bashrc +++ b/ansible/roles/hubs/templates/bashrc @@ -23,14 +23,9 @@ workon() { cd {{ hubs_code_dir }} } -hup() { - source {{ hubs_venv_dir }}/bin/activate - pushd {{ hubs_code_dir }} - FLASK_DEBUG=1 flask run --host 0.0.0.0 --port 5000 -} +alias hup="pushd ~ ; honcho start ; popd" hreset() { - source {{ hubs_venv_dir }}/bin/activate {% if hubs_db_type == "postgresql" %} sudo -u postgres dropdb hubs sudo -u postgres createdb -O hubs hubs @@ -39,7 +34,6 @@ hreset() { {% endif %} rm {{ hubs_var_dir }}/cache.db pushd {{ hubs_code_dir }} - python populate.py + {{ hubs_venv_dir }}/bin/python populate.py popd - deactivate } diff --git a/ansible/roles/hubs/templates/honcho-env b/ansible/roles/hubs/templates/honcho-env new file mode 100644 index 0000000..352c551 --- /dev/null +++ b/ansible/roles/hubs/templates/honcho-env @@ -0,0 +1,3 @@ +FLASK_DEBUG=1 +FLASK_APP={{ hubs_code_dir }}/hubs/app.py +HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py diff --git a/ansible/roles/hubs/templates/honcho-procfile b/ansible/roles/hubs/templates/honcho-procfile new file mode 100644 index 0000000..ee8a451 --- /dev/null +++ b/ansible/roles/hubs/templates/honcho-procfile @@ -0,0 +1,6 @@ +web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask run --host 0.0.0.0 --port 5000 +triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage +worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker +sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac +fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub +js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml index a691079..f24cfa8 100644 --- a/ansible/vagrant-playbook.yml +++ b/ansible/vagrant-playbook.yml @@ -27,3 +27,12 @@ state: link path: /home/{{ main_user }}/devel src: /srv/hubs/fedora-hubs + + - name: Make a symlink to the Honcho files + file: + state: link + path: /home/{{ main_user }}/{{ item }} + src: /srv/hubs/{{ item }} + with_items: + - Procfile + - .env From ac53197aba95e78d76828b185b17c97f4e36d754 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jul 30 2017 14:14:27 +0000 Subject: [PATCH 2/4] Get widget parameter defaults more dynamically --- diff --git a/hubs/models.py b/hubs/models.py index 0a58e40..f894c22 100644 --- a/hubs/models.py +++ b/hubs/models.py @@ -28,6 +28,7 @@ import logging import operator import os import random +from collections import defaultdict import bleach import sqlalchemy as sa @@ -346,12 +347,18 @@ class HubConfig(BASE): return 12 - self.left_width -def _widget_config_default(context): - widget_name = context.current_parameters['plugin'] - widget = hubs.widgets.registry[widget_name] - return json.dumps(dict([ - (param.name, param.default) for param in widget.get_parameters() - ])) +class SpecificDefaultDict(defaultdict): + """A more specific version of defaultdict. + + This class behaves like defaultdict, but calls the ``default_factory`` with + the key as first argument. + """ + + def __missing__(self, key): + if self.default_factory is None: + return super(SpecificDefaultDict, self).__missing__(key) + self[key] = self.default_factory(key) + return self[key] class Widget(ObjectAuthzMixin, BASE): @@ -364,7 +371,7 @@ class Widget(ObjectAuthzMixin, BASE): plugin = sa.Column(sa.String(50), nullable=False) created_on = sa.Column(sa.DateTime, default=datetime.datetime.utcnow) hub_id = sa.Column(sa.String(50), sa.ForeignKey('hubs.name')) - _config = sa.Column(sa.Text, default=_widget_config_default) + _config = sa.Column(sa.Text, default="{}") index = sa.Column(sa.Integer, nullable=False) left = sa.Column(sa.Boolean, nullable=False, default=False) @@ -388,7 +395,17 @@ class Widget(ObjectAuthzMixin, BASE): @property def config(self): - return json.loads(self._config) + def get_default(key): + for param in self.module.get_parameters(): + if key == param.name: + break + else: + raise KeyError("No such parameter") + return param.default + + value = SpecificDefaultDict(get_default) + value.update(json.loads(self._config)) + return value @config.setter def config(self, config): diff --git a/hubs/widgets/base.py b/hubs/widgets/base.py index aa8c5b8..7f35b37 100644 --- a/hubs/widgets/base.py +++ b/hubs/widgets/base.py @@ -9,6 +9,7 @@ import six from importlib import import_module from .caching import CachedFunction from .view import WidgetView +from .validators import Validator log = logging.getLogger(__name__) @@ -46,6 +47,9 @@ class WidgetParameter(object): setattr(self, name, kwargs.pop(name, None)) for name in kwargs: raise TypeError("Invalid attribute: %s" % name) + # Set default validator + if self.validator is None: + self.validator = Validator class Widget(object): From 96a5824082b3b0b98c22eb1607e2fd2c07de610b Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jul 30 2017 14:15:19 +0000 Subject: [PATCH 3/4] Don't force overriding get_context() --- diff --git a/hubs/widgets/view.py b/hubs/widgets/view.py index 3794bb3..fce34b3 100644 --- a/hubs/widgets/view.py +++ b/hubs/widgets/view.py @@ -69,7 +69,7 @@ class WidgetView(View): Args: instance (hubs.models.Widget): the widget instance. """ - raise NotImplementedError + return {} def get_template(self): """ From adefa8a86ddc2a748ade65184262e4e6d94d71f4 Mon Sep 17 00:00:00 2001 From: Aurélien Bompard Date: Jul 31 2017 11:27:11 +0000 Subject: [PATCH 4/4] Widget cache must be rebuilt on config change --- diff --git a/hubs/widgets/caching.py b/hubs/widgets/caching.py index 02abde3..10b7903 100644 --- a/hubs/widgets/caching.py +++ b/hubs/widgets/caching.py @@ -55,6 +55,7 @@ class CachedFunction(object): return "|".join([ str(self.instance.idx), self.__class__.__name__, + str(dict(self.instance.config)), ]).encode('utf-8') def get_data(self):