From ece01ea7c48861a342ab22f58d16652f44f960d4 Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Sep 29 2016 13:42:27 +0000 Subject: Add a Vagrantfile and Ansible role This is a first pass at a Vagrantfile and Ansible role to automate the setup of a development environment. This also adds a basic dev guide document which, at the moment, only includes Vagrant instructions. I plan to organize the documentation and update the README in a separate commit. closes #254 Signed-off-by: Jeremy Cline --- diff --git a/.gitignore b/.gitignore index 096613a..357e7b0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,10 @@ scratch/ npm-debug.log client_secrets.json hubs.js + +# Vagrant and Ansible files +Vagrantfile +.dnf-cache +*.retry +*.pem +config diff --git a/Vagrantfile.example b/Vagrantfile.example new file mode 100644 index 0000000..7fff78d --- /dev/null +++ b/Vagrantfile.example @@ -0,0 +1,63 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "fedora/24-cloud-base" + + # Forward traffic on the host to the development server on the guest + config.vm.network "forwarded_port", guest: 5000, host: 5000 + + # Comment out if you don't want Vagrant to add and remove entries from /etc/hosts for each VM. + # requires the vagrant-hostmanager plugin to be installed + if Vagrant.has_plugin?("vagrant-hostmanager") + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + end + + # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs + # plugin). By default it rsyncs the current working directory to /vagrant. We disable this + # in favor of SSHFS. + # + # If you would prefer to use NFS to share the directory uncomment this and configure NFS + #config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false + config.vm.synced_folder ".", "/vagrant", disabled: true + config.vm.synced_folder ".", "/home/vagrant/devel", type: "sshfs", sshfs_opts_append: "-o nonempty" + + # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) + # you can create a local directory and share it to the guest's DNF cache. The directory needs to + # exist, so create it before you uncomment the line below. + #config.vm.synced_folder ".dnf-cache", "/var/cache/dnf", type: "sshfs", sshfs_opts_append: "-o nonempty" + + # Comment this line if you would like to disable the automatic update during provisioning + config.vm.provision "shell", inline: "sudo dnf upgrade -y" + + # Install packages required for Ansible and configure everything with ansible + config.vm.provision "shell", inline: "sudo dnf -y install python2-dnf libselinux-python" + config.vm.provision "ansible" do |ansible| + ansible.playbook = "ansible/vagrant-playbook.yml" + end + + + # Create the "pagure" box + config.vm.define "hubs" do |hubs| + hubs.vm.host_name = "hubs-dev.example.com" + + hubs.vm.provider :libvirt do |domain| + # Season to taste + domain.cpus = 4 + domain.graphics_type = "spice" + domain.memory = 2048 + domain.video_type = "qxl" + + # Uncomment the following line if you would like to enable libvirt's unsafe cache + # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all + # fsync() calls from the guest. Only do this if you are comfortable with the possibility of + # your development guest becoming corrupted (in which case you should only need to do a + # vagrant destroy and vagrant up to get a new one). + # + #domain.volume_cache = "unsafe" + end + end +end diff --git a/ansible/roles/hubs-dev/files/bashrc b/ansible/roles/hubs-dev/files/bashrc new file mode 100644 index 0000000..615b5df --- /dev/null +++ b/ansible/roles/hubs-dev/files/bashrc @@ -0,0 +1,39 @@ +# .bashrc + +# Source global definitions +if [ -f /etc/bashrc ]; then + . /etc/bashrc +fi + +# Uncomment the following line if you don't like systemctl's auto-paging feature: +# export SYSTEMD_PAGER= + +# User specific aliases and functions +# If adding new functions to this file, note that you can add help text to the function +# by defining a variable with name __help containing the help text + +# Set up virtualenvwrapper +export WORKON_HOME=$HOME/.virtualenvs +export PIP_VIRTUALENV_BASE=$WORKON_HOME +export VIRTUALENV_USE_DISTRIBUTE=true +export PIP_RESPECT_VIRTUALENV=true +source /usr/bin/virtualenvwrapper.sh + +export HUBS_CONFIG=~/devel/config + + +hup() { + workon hubs + pushd ~/devel/ + python ./runserver.py --host 0.0.0.0 +} + +hreset() { + workon hubs + rm /var/tmp/hubs.db + rm /var/tmp/fedora-hubs-cache.db + pushd ~/devel/ + python populate.py + popd + deactivate +} diff --git a/ansible/roles/hubs-dev/files/datanommer.py b/ansible/roles/hubs-dev/files/datanommer.py new file mode 100644 index 0000000..30de6fb --- /dev/null +++ b/ansible/roles/hubs-dev/files/datanommer.py @@ -0,0 +1,4 @@ +config = { + 'datanommer.enabled': True, + 'datanommer.sqlalchemy.url': 'postgres://datanommer@localhost/datanommer', +} diff --git a/ansible/roles/hubs-dev/files/hubs_config b/ansible/roles/hubs-dev/files/hubs_config new file mode 100644 index 0000000..83417df --- /dev/null +++ b/ansible/roles/hubs-dev/files/hubs_config @@ -0,0 +1,4 @@ +# Enter any hubs configuratio here + +# Allow the cookie to be sent of http since we work on localhost +OIDC_ID_TOKEN_COOKIE_SECURE = False diff --git a/ansible/roles/hubs-dev/files/motd b/ansible/roles/hubs-dev/files/motd new file mode 100644 index 0000000..f3d92ab --- /dev/null +++ b/ansible/roles/hubs-dev/files/motd @@ -0,0 +1,18 @@ + +Welcome to the Fedora Hubs development environment! + +Here are some tips: + +* The code for Fedora Hubs is located at ~/devel/ + +* You can type `workon hubs` to enter a configured Python virtualenv + +* Run `hup` to start gunicorn + +* Run `hreset` to delete the database and repopulate it + +Once you run `hup` you can navigate to http://localhost:5000/ +in your browser. + +Happy hacking! + diff --git a/ansible/roles/hubs-dev/files/pg_hba.conf b/ansible/roles/hubs-dev/files/pg_hba.conf new file mode 100644 index 0000000..fb47623 --- /dev/null +++ b/ansible/roles/hubs-dev/files/pg_hba.conf @@ -0,0 +1,14 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 trust diff --git a/ansible/roles/hubs-dev/handlers/main.yml b/ansible/roles/hubs-dev/handlers/main.yml new file mode 100644 index 0000000..5d4ca3e --- /dev/null +++ b/ansible/roles/hubs-dev/handlers/main.yml @@ -0,0 +1,5 @@ +- name: restart postgresql + service: name=postgresql state=restarted + +- name: restart fedmsg-hub + service: name=fedmsg-hub state=restarted diff --git a/ansible/roles/hubs-dev/tasks/main.yml b/ansible/roles/hubs-dev/tasks/main.yml new file mode 100755 index 0000000..c9add18 --- /dev/null +++ b/ansible/roles/hubs-dev/tasks/main.yml @@ -0,0 +1,179 @@ +--- +- name: Install helpful development packages + dnf: name={{ item }} state=present + with_items: + - git + - vim-enhanced + +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - postgresql + - postgresql-devel + - python-sphinx + - python-virtualenvwrapper + - python2-devel + - python3-devel + - redhat-rpm-config + - sqlite-devel + +- name: Install the distribution versions of requirements.txt + dnf: name={{ item }} state=present + with_items: + - python-arrow + - python-bleach + - python-datanommer-models + - python-decorator + - python-dogpile-cache + - python-fedmsg-core + - python-fedmsg-meta-fedora-infrastructure + - python-flask + - python3-flask-oidc + - python-fmn-lib + - python-fmn-rules + - python-gunicorn + - python-html5lib + - python-munch + - python-psycopg2 + - pytz + - python-sqlalchemy + - python-markdown + - python-pkgwat-api + - python-six + - python-pygments + - python-pygments-markdown-lexer + - python-retask + +- name: Install packages for datanommer + dnf: name={{ item }} state=present + with_items: + - datanommer-commands + - fedmsg-hub + - npm + - postgresql-server + - python-datanommer-consumer + - python-fedmsg-meta-fedora-infrastructure + - python-psycopg2 + + +# Add various helpful configuration files +- name: Install a custom bashrc + copy: src=bashrc dest=/home/{{ ansible_env.SUDO_USER }}/.bashrc + +- name: Install the message of the day + copy: src=motd dest=/etc/motd + + +# Set up Postgres, create the necessary databases, and start up datanommer +- name: Set up postgresql database + command: postgresql-setup --initdb + args: + creates: /var/lib/pgsql/data/base + +- name: Set up postgresql access rules to allow local access + copy: + src: pg_hba.conf + dest: /var/lib/pgsql/data/pg_hba.conf + owner: postgres + group: postgres + mode: 0600 + notify: restart postgresql + +- name: Start and enable postgresql + service: name=postgresql state=started enabled=yes + +- name: Set up datanommer DB user + postgresql_user: + name: datanommer + role_attr_flags: SUPERUSER,LOGIN + +- name: Create datanommer database + postgresql_db: + name: datanommer + owner: datanommer + +- name: Set up datanommer + copy: src=datanommer.py dest=/etc/fedmsg.d/datanommer.py + notify: restart fedmsg-hub + +# TODO unfortunately this always runs as there doesn't appear to be +# an easy way to check if it needs to run +- name: Create datanommer database tables + command: datanommer-create-db + +- name: Start and enable fedmsg-hub + service: name=fedmsg-hub state=started enabled=yes + + +# Set up the Python development environment +- name: Install Fedora Hubs requirements.txt into hubs virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + requirements: /home/{{ ansible_env.SUDO_USER }}/devel/requirements.txt + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + +- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + requirements: /home/{{ ansible_env.SUDO_USER }}/devel/requirements.txt + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + +- name: Install bleach into hubs virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + name: bleach + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + +- name: Install gunicorn into hubs virtualenv + become_user: "{{ ansible_env.SUDO_USER }}" + pip: + name: gunicorn + virtualenv: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/ + +- name: Update httplib2 trust store + copy: + src: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + dest: /home/{{ ansible_env.SUDO_USER }}/.virtualenvs/hubs/lib/python2.7/site-packages/httplib2/cacerts.txt + remote_src: True + +- name: Add a basic Hubs configuration file + copy: src=hubs_config dest=/home/{{ ansible_env.SUDO_USER }}/devel/config + become_user: "{{ ansible_env.SUDO_USER }}" + +- name: Configure application to authenticate with iddev.fedorainfracloud.org + shell: > + source ~/.bashrc && + workon hubs && + oidc-register https://iddev.fedorainfracloud.org/ http://localhost:5000 + become_user: "{{ ansible_env.SUDO_USER }}" + args: + creates: client_secrets.json + chdir: "/home/{{ ansible_env.SUDO_USER }}/devel/" + +- name: Populate the Fedora Hubs database + shell: > + source ~/.bashrc && + workon hubs && + python populate.py + become_user: "{{ ansible_env.SUDO_USER }}" + args: + creates: /var/tmp/hubs.db + chdir: "/home/{{ ansible_env.SUDO_USER }}/devel/" + +# Set up JavaScript requirements +- name: Install npm packages + command: npm install + become_user: "{{ ansible_env.SUDO_USER }}" + args: + creates: node_modules + chdir: /home/{{ ansible_env.SUDO_USER }}/devel/hubs/static/client + +- name: Build JavaScript assests + command: node_modules/.bin/webpack + become_user: "{{ ansible_env.SUDO_USER }}" + args: + chdir: /home/{{ ansible_env.SUDO_USER }}/devel/hubs/static/client diff --git a/ansible/vagrant-playbook.yml b/ansible/vagrant-playbook.yml new file mode 100644 index 0000000..14cbf9f --- /dev/null +++ b/ansible/vagrant-playbook.yml @@ -0,0 +1,7 @@ +--- +- hosts: all + become: true + become_method: sudo + vars: + roles: + - hubs-dev diff --git a/dev-guide.rst b/dev-guide.rst new file mode 100644 index 0000000..fda1d46 --- /dev/null +++ b/dev-guide.rst @@ -0,0 +1,44 @@ +Developer Setup +=============== + +This guide should get you up and running with a development environment. +The simpliest way to stand up a development environment is to use +`Vagrant `_, but it is also easy to set +up a development environment without Vagrant. + +Vagrant +^^^^^^^ + +A ``Vagrantfile`` called ``Vagrantfile.example`` is provided in the root of the +`Fedora Hub repository `_. To use it, simply +copy ``Vagrantfile.example`` to ``Vagrantfile`` and modify it as you see fit. +All settings are documented inline. + +If you do not have Vagrant set up, follow these steps: + +#. Install Ansible, Vagrant, and several helpful Vagrant plugins:: + + $ sudo dnf install ansible vagrant-libvirt vagrant-sshfs vagrant-hostmanager + +#. Create and configure the virtual machine:: + + $ vagrant up --provider=libvirt + $ vagrant reload # Reboot the machine to use the latest kernel, etc. + +#. You can now use ssh to connect to the machine:: + + $ vagrant ssh + +#. If you ever want to re-run the provisioning scripts, you can:: + + $ vagrant provision + +#. If you decide you want a fresh development environment, simply destroy your + virtual machine and recreate it:: + + $ vagrant destroy && vagrant up + +You should now be able to point the browser in your host at http://localhost:5000/ +and use Fedora Hubs after you start the development server in the virtual machine. + +Happy hacking!