Merge pull request #26 from lgierth/gateway

gateway: import solarnet ansible scripts

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Juan Batiz-Benet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

solarnet/.gitignore

@@ -0,0 +1,2 @@
+venv/
+secrets.yml

solarnet/Makefile

@@ -0,0 +1,22 @@
+venv/bin/activate:
+	test -d venv || virtualenv venv
+
+venv/bin/ansible: venv/bin/activate
+	. venv/bin/activate && pip install -r requirements.txt
+
+deps: venv/bin/ansible
+
+cake: venv/bin/ansible ipfs_ref
+	. venv/bin/activate && ansible-playbook solarnet.yml
+
+IPFS_REPO = https://github.com/ipfs/go-ipfs.git
+ipfs_ref : IPFS_REF = $(shell git ls-remote -q $(IPFS_REPO) master | awk "{print \$$1}")
+ipfs_ref:
+	sed -i'.bak' "s/ipfs_ref:.*/ipfs_ref: $(IPFS_REF)/" roles/ipfs/vars/main.yml
+	rm roles/ipfs/vars/main.yml.bak
+
+CJDNS_REPO = https://github.com/hyperboria/cjdns.git
+cjdns_ref : cjdns_REF = $(shell git ls-remote -q $(CJDNS_REPO) master | awk "{print \$$1}")
+cjdns_ref:
+	sed -i'.bak' "s/cjdns_ref:.*/cjdns_ref: $(cjdns_REF)/" roles/cjdns/vars/main.yml
+	rm roles/cjdns/vars/main.yml.bak

solarnet/README.md

@@ -0,0 +1,96 @@
+# solarnet
+
+This network runs the IPFS bootstrap + gateway nodes. There are 9 machines, all on Digital Ocean in various different data centers across the world. See the `hosts` file.
+
+## Node Setup
+
+Each solarnet computer runs one ipfs process. it handles both bootstrap and http://gateway.ipfs.io.
+
+They have:
+
+- docker - runs things in a uniform, clean environment.
+- nginx - which proxies over to gateway.
+- ipfs daemon - which runs the gateway.
+
+### nginx
+
+nginx absorbs all the outside requests coming to the gateways. This allows us to:
+- deal with HTTP traffic better (nginx is a beast!)
+- not have to worry about a number of attacks on HTTP servers
+- use proxy-pass to an ipfs gateway
+- route traffic to other gateways if local gateway ipfs program crashes
+
+### ipfs daemon
+
+Gets its repo mounted as a volume, and exposes the ports 4001 (swarm), 5001 (API), and 8080 (gateway) to the host.
+
+## Getting Started
+
+### ansible
+
+This repository uses ansible for _almost_ everything.  ansible is fairly nice, but can also be annoying. it comes well recommended by others, though none of us are experts with it. (@jbenet would prefer to eiter use shell scripts or ansible, but probably not other tools _more_ complicated than ansible (e.g. chef / puppet), [@lgierth agrees](https://github.com/lgierth/provsn)).
+
+```sh
+# implicit dependencies: virtualenv pip
+# install stuff in requirements.txt
+$ make deps
+
+# activate the virtualenv
+$ . venv/bin/activate
+
+# see if it works
+(venv)$ which ansible
+(venv)$ ansible solarnet -a 'docker ps'
+```
+
+## Deploying
+
+Please commit the changed `roles/ipfs/vars/main.yml` when deploying an update!
+
+```sh
+# update the ipfs role's commit reference
+$ make ipfs_ref
+
+# deploy to all solarnet hosts
+(venv)$ ansible-playbook -l solarnet solarnet.yml
+
+# deploy one host at a time
+(venv)$ ansible-playbook -f 1 -l solarnet solarnet.yml
+
+# deploy only pluto.i.ipfs.io
+(venv)$ ansible-playbook -l pluto solarnet.yml
+```
+
+or simply:
+
+```sh
+$ make cake
+```
+
+### Restarting
+
+```sh
+(venv)$ ansible solarnet -a 'docker restart ipfs'
+```
+
+## Troubleshooting
+
+### disk space
+
+A couple of things fill up the disk. These should be fixed, or the cleanup
+one-liners be automated.
+
+```sh
+# docker's container output logs. restart container so it reopns the logs.
+(venv)$ ansible solarnet -f 1 -m shell -a 'rm -v /var/lib/docker/containers/*/*-json.log ; docker restart ipfs_master'
+
+# old docker containers and images
+(venv)$ ansible solarnet -m shell -a 'docker rm $(docker ps -f "status=exited" -aq) ; docker rmi $(docker images -f "dangling=true" -aq)'
+
+# all of the above
+(venv)$ ./cleanup.sh
+```
+
+### unsupported parameter for module: restart_policy
+
+You're probably using the system version of Ansible, and it is outdated (< 1.9). Make sure to follow the Ansible setup steps above, and load the virtualenv. This will load a working version of Ansible.

solarnet/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+remote_user         = root
+hostfile            = hosts
+host_key_checking   = False

solarnet/cleanup.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ansible solarnet -f 4 -m shell -a 'rm -v /var/lib/docker/containers/*/*-json.log ; docker restart ipfs ; sleep 2 ; docker rm $(docker ps -f "status=exited" -aq) ; docker rmi $(docker images -f "dangling=true" -aq)'

solarnet/files/maybe-restart.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+if [ "$#" -lt 2 ]; then
+  echo "usage: $0 <0-100> <cmd> [<args>...]"
+  echo "  run command <cmd> with a given percent probability"
+  exit 1
+fi
+
+chance=$1
+shift
+
+random=`hexdump -n 2 -e '/2 "%u"' /dev/urandom`
+
+[ `expr $random % 100` -lt "$chance" ] && eval "$@"

solarnet/hosts

@@ -0,0 +1,13 @@
+[solarnet]
+pluto       ansible_ssh_host=104.236.179.241
+neptune     ansible_ssh_host=104.236.176.52
+uranus      ansible_ssh_host=162.243.248.213
+saturn      ansible_ssh_host=128.199.219.111
+jupiter     ansible_ssh_host=104.236.151.122
+venus       ansible_ssh_host=104.236.76.40
+earth       ansible_ssh_host=178.62.158.247
+mercury     ansible_ssh_host=178.62.61.185
+
+; managed by whyrusleeping
+[whyrunet]
+mars        ansible_ssh_host=104.131.131.82

solarnet/playbooks/domains.yaml

@@ -0,0 +1,108 @@
+---
+- hosts: jenkins
+  vars:
+      do:
+          DO_API_KEY: "{{ lookup('env','DIGITAL_OCEAN_API_KEY') }}"
+          DO_CLIENT_ID: "{{  lookup('env','DIGITAL_OCEAN_CLIENT_ID') }}"
+  tasks:
+    - name: "install dopy"
+      pip: name=dopy
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: uranus.nyc.srv.protocol-dev.com
+        ip: 162.243.248.213
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: pluto.sfo.srv.protocol-dev.com
+        ip: 104.236.179.241
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: neptune.sfo.srv.protocol-dev.com
+        ip: 104.236.176.52
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: saturn.sg.srv.protocol-dev.com
+        ip: 128.199.219.111
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: jupiter.sfo.srv.protocol-dev.com
+        ip: 104.236.175.101
+
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: uranus.srv.protocol-dev.com
+        ip: 162.243.248.213
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: pluto.srv.protocol-dev.com
+        ip: 104.236.179.241
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: neptune.srv.protocol-dev.com
+        ip: 104.236.176.52
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: saturn.srv.protocol-dev.com
+        ip: 128.199.219.111
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: jupiter.srv.protocol-dev.com
+        ip: 104.236.175.101
+
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: uranus.i.ipfs.io
+        ip: 162.243.248.213
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: pluto.i.ipfs.io
+        ip: 104.236.179.241
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: neptune.i.ipfs.io
+        ip: 104.236.176.52
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: saturn.i.ipfs.io
+        ip: 128.199.219.111
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: jupiter.i.ipfs.io
+        ip: 104.236.175.101
+
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: mercury.i.ipfs.io
+        ip: 178.62.61.185
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: earth.i.ipfs.io
+        ip: 178.62.158.247
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: venus.i.ipfs.io
+        ip: 104.236.76.40
+
+    - environment: do
+      digital_ocean_domain:
+        state: present
+        name: mars.i.ipfs.io
+        ip: 104.131.131.82

solarnet/requirements.txt

@@ -0,0 +1,11 @@
+ansible==1.9.2
+Jinja2==2.7.3
+MarkupSafe==0.23
+PyYAML==3.11
+bzr==2.6.0
+ecdsa==0.11
+httplib2==0.9
+mercurial==2.9.1
+paramiko==1.15.2
+pycrypto==2.6.1
+wsgiref==0.1.2

solarnet/roles/cjdns/tasks/main.yml

@@ -0,0 +1,45 @@
+---
+- name: check current cjdns ref
+  shell: "cat /opt/cjdns.ref | grep {{ cjdns_ref }}"
+  ignore_errors: true
+  register: cjdns_ref_present
+
+- name: clone cjdns.git
+  git:
+    repo: https://github.com/hyperboria/cjdns.git
+    dest: /opt/cjdns
+    version: "{{ cjdns_ref }}"
+
+- name: build cjdns
+  command: ./do chdir=/opt/cjdns
+  when: "cjdns_ref_present.rc != 0"
+
+- name: install cjdroute
+  copy:
+    src: /opt/cjdns/cjdroute
+    dest: /usr/bin/cjdroute
+    mode: 0755
+  when: "cjdns_ref_present.rc != 0"
+
+- name: install cjdroute.conf
+  template:
+    src: cjdroute.conf.j2
+    dest: /etc/cjdroute.conf
+    mode: 0400
+  register: cjdroute_conf
+
+- name: install upstart script
+  copy:
+    src: /opt/cjdns/contrib/upstart/cjdns.conf
+    dest: /etc/init/cjdns.conf
+    mode: 0644
+
+- name: restart cjdroute
+  service:
+    name: cjdns
+    state: restarted
+  when: "cjdns_ref_present.rc != 0 or cjdroute_conf.changed"
+
+- name: record new cjdns ref
+  shell: "echo {{ cjdns_ref }} > /opt/cjdns.ref"
+  when: "cjdns_ref_present.rc != 0"

solarnet/roles/cjdns/templates/cjdroute.conf.j2

@@ -0,0 +1,38 @@
+{
+    "privateKey": "{{ cjdns_identities[inventory_hostname].private_key }}",
+    "admin": {
+        "bind": "{{ cjdns_admin_address }}:{{ cjdns_admin_port }}",
+        "password": "{{ cjdns_admin_password }}"
+    },
+    "interfaces": {
+        "UDPInterface": [
+{% for interface in cjdns_udp_interfaces %}
+        {
+            "bind": "{{ interface.bind }}",
+            "connectTo": {
+{% for peer in interface.peers %}
+                "{{ peer.connect_to }}": {
+                    "publicKey": "{{ peer.public_key }}",
+                    "password": "{{ peer.password }}"
+                },
+{% endfor %}
+            }
+        },
+{% endfor %}
+        ]
+    },
+    "router": {
+        "interface": {
+            "type": "TUNInterface",
+            "tunDevice": "{{ cjdns_tun_interface }}"
+        }
+    },
+    "security": [
+        { "setuser": "nobody", "keepNetAdmin": 1 },
+        { "chroot": "/var/run/" },
+        { "nofiles": 0 },
+        { "noforks": 1 },
+        { "seccomp": 1 },
+        { "setupComplete": 1 }
+    ]
+}

solarnet/roles/cjdns/vars/main.yml

@@ -0,0 +1,7 @@
+---
+cjdns_ref: 3d2ce14e7c45cb50969853c9bfd842a33720c922
+cjdns_admin_address: 127.0.0.1
+cjdns_admin_port: 11234
+cjdns_admin_password: thepassword
+cjdns_tun_interface: tun0
+cjdns_udp_interfaces: []