Merge pull request #272 from asteris-llc/feature/example-elk

example: elasticsearch, kibana, and filebeat ~elk

.gitignore

@@ -40,4 +40,5 @@ docs_source/resources.mk
 docs_source/public
 
 # examples
-terraform.tfstate*
+terraform.tfstate*
+.vagrant

examples/elk/README.md

@@ -0,0 +1,63 @@
+# converge-elk
+
+The [ELK stack](https://www.elastic.co/webinars/introduction-elk-stack) is
+traditionally composed of
+[Elasticsearch](https://www.elastic.co/products/elasticsearch),
+[Logstash](https://www.elastic.co/products/logstash), and
+[Kibana](https://www.elastic.co/products/kibana) and is a great solution for
+collecting, searching, and visualizing logs. In this example, we are configuring
+a docker-based ELK stack. However, since this is a single-node demonstration, we
+are using [Filebeat](https://www.elastic.co/products/beats/filebeat) instead of
+Logstash for the log collection component.
+
+## Usage
+
+### Vagrant
+
+Just run `vagrant up`!
+
+After Vagrant provisioning completes, you should have a working Kibana instance
+backed by Elasticsearch. Filebeat is installed on the Vagrant host and is
+configured to send logs to Elasticsearch. You should be able to access the
+Kibana web interface at [http://localhost:5601](http://localhost:5601).
+
+### Terraform (AWS)
+
+You must have a version of the
+[Converge Terraform provisioner](https://github.com/ChrisAubuchon/terraform-provisioner-converge)
+built and configured as a plugin for Terraform:
+
+```shell
+$ cat ~/.terraformrc
+provisioners {
+  converge = "/path/to/terraform-provisioner-converge"
+}
+```
+
+You must have also set valid
+[AWS credentials](https://www.terraform.io/docs/providers/aws/index.html)
+(`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) in your environment. Then you
+can run:
+
+```
+terraform apply
+```
+
+After provisioning completes, you should be able to retrieve the url for the
+Kibana interface by running:
+
+```shell
+echo "http://$(terraform output ip):5601/"
+```
+
+## Graphs
+
+This is the visualization of the graph that Converge applies to the system.
+
+![elk graph](./graphs/elk.png)
+
+## Warning
+
+When deploying via Terraform, Kibana will be publicly accessible on port 5601
+without authentication. You can adjust the security group in `main.tf` to change
+this behavior.

examples/elk/Vagrantfile

@@ -0,0 +1,20 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+converge_version = "0.2.0-beta1"
+release_url = "https://github.com/asteris-llc/converge/releases/download/#{converge_version}/converge_#{converge_version}_linux_amd64.tar.gz"
+
+converge_script = <<SCRIPT
+cd /tmp
+curl -SL #{release_url} -o converge.tar.gz
+tar -xzvf converge.tar.gz
+mv converge /usr/local/bin
+sudo /usr/local/bin/converge apply --local --log-level=info /vagrant/converge/elk.hcl
+SCRIPT
+
+Vagrant.configure("2") do |config|
+  config.vm.box = "centos/7"
+  config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ".git/"
+  config.vm.network "forwarded_port", guest: 5601, host: 5601
+  config.vm.provision "shell", inline: converge_script
+end

examples/elk/converge/docker.hcl

@@ -0,0 +1,52 @@
+param "docker-package" {
+  default = "docker-engine"
+}
+
+param "docker-service" {
+  default = "docker"
+}
+
+param "docker-group" {
+  default = "docker"
+}
+
+param "user-name" {
+  default = "vagrant"
+}
+
+file.content "docker-repo" {
+  destination = "/etc/yum.repos.d/docker.repo"
+
+  content = <<EOF
+[dockerrepo]
+name=Docker Repository
+baseurl=https://yum.dockerproject.org/repo/main/centos/7/
+enabled=1
+gpgcheck=1
+gpgkey=https://yum.dockerproject.org/gpg
+EOF
+}
+
+task "docker-install" {
+  check   = "yum list installed {{param `docker-package`}}"
+  apply   = "yum makecache; yum install -y {{param `docker-package`}}"
+  depends = ["file.content.docker-repo"]
+}
+
+task "docker-user-group" {
+  check   = "groups {{param `user-name`}} | grep -i {{param `docker-group`}}"
+  apply   = "usermod -aG {{param `docker-group`}} {{param `user-name`}}"
+  depends = ["task.docker-install"]
+}
+
+task "docker-enable" {
+  check   = "systemctl is-enabled {{param `docker-service`}}"
+  apply   = "systemctl enable {{param `docker-service`}}"
+  depends = ["task.docker-user-group"]
+}
+
+task "docker-start" {
+  check   = "systemctl is-active {{param `docker-service`}}"
+  apply   = "systemctl start {{param `docker-service`}}"
+  depends = ["task.docker-enable"]
+}

examples/elk/converge/elk.hcl

@@ -0,0 +1,114 @@
+param "user-name" {
+  default = "vagrant"
+}
+
+module "packages.hcl" "packages" {}
+
+module "docker.hcl" "docker" {
+  params = {
+    user-name = "{{param `user-name`}}"
+  }
+  depends = ["module.packages"]
+}
+
+param "elasticsearch-data-directory" {
+  default = "/data/elasticsearch"
+}
+
+param "filebeat-service" {
+  default = "filebeat"
+}
+
+task "filebeat-install" {
+  check   = "yum list installed filebeat"
+  apply   = "rpm -ivh https://download.elastic.co/beats/filebeat/filebeat-1.3.0-x86_64.rpm"
+  depends = ["module.docker"]
+}
+
+file.content "filebeat-yml" {
+  destination = "/etc/filebeat/filebeat.yml"
+
+  content = <<EOF
+filebeat:
+  prospectors:
+    - paths:
+        - /var/log/*.log
+        - /var/log/messages
+      input_type: log
+  registry_file: /var/lib/filebeat/registry
+output:
+  elasticsearch:
+    hosts: ["localhost:9200"]
+EOF
+
+  depends = ["task.filebeat-install"]
+}
+
+task "filebeat-enable" {
+  check   = "systemctl is-enabled {{param `filebeat-service`}}"
+  apply   = "systemctl enable {{param `filebeat-service`}}"
+  depends = ["file.content.filebeat-yml"]
+}
+
+task.query "elasticsearch-wait" {
+  query = <<EOF
+MAX_SECONDS=60
+while /bin/true
+do
+    status=$(curl -s 'http://localhost:9200/_cluster/health' 2>/dev/null | jq -r .status)
+    if [ "$status" == "yellow" ] || [ "$status" == "green" ] ; then
+        exit 0
+    fi
+    [[ "$SECONDS" -ge "$MAX_SECONDS" ]] && exit 1
+done
+EOF
+
+  depends = ["docker.container.elasticsearch-container"]
+}
+
+task "filebeat-elasticsearch-template" {
+  check   = "[[  \"$(curl 'http://localhost:9200/_template/filebeat' 2>/dev/null)\" != \"{}\" ]] || exit 1"
+  apply   = "curl -XPUT 'http://localhost:9200/_template/filebeat' -d@/etc/filebeat/filebeat.template.json 2>/dev/null"
+  depends = ["task.filebeat-enable", "docker.container.elasticsearch-container", "task.query.elasticsearch-wait"]
+}
+
+task "filebeat-start" {
+  check   = "systemctl is-active {{param `filebeat-service`}}"
+  apply   = "systemctl start {{param `filebeat-service`}}"
+  depends = ["task.filebeat-enable", "docker.container.elasticsearch-container"]
+}
+
+file.directory "elasticsearch-data-directory" {
+  destination = "{{param `elasticsearch-data-directory`}}"
+  create_all = true
+}
+
+docker.image "elasticsearch-image" {
+  name    = "elasticsearch"
+  tag     = "2.4.0"
+  depends = ["module.docker"]
+}
+
+docker.container "elasticsearch-container" {
+  name    = "elasticsearch"
+  image   = "{{lookup `docker.image.elasticsearch-image.name`}}:{{lookup `docker.image.elasticsearch-image.tag`}}"
+  command = ["elasticsearch", "-Des.insecure.allow.root=true"]
+  ports   = ["127.0.0.1:9200:9200"]
+  volumes = ["{{param `elasticsearch-data-directory`}}:/usr/share/elasticsearch/data"]
+  force   = "true"
+  depends = ["file.directory.elasticsearch-data-directory"]
+}
+
+docker.image "kibana-image" {
+  name    = "kibana"
+  tag     = "4.6.0"
+  depends = ["module.docker"]
+}
+
+docker.container "kibana-container" {
+  name  = "kibana"
+  image = "{{lookup `docker.image.kibana-image.name`}}:{{lookup `docker.image.kibana-image.tag`}}"
+  ports = ["5601:5601"]
+  links = ["{{lookup `docker.container.elasticsearch-container.name`}}:elasticsearch"]
+  force = "true"
+}

examples/elk/converge/packages.hcl

@@ -0,0 +1,10 @@
+task "epel-install" {
+  check = "test -f /etc/yum.repos.d/epel.repo"
+  apply = "yum makecache; yum install -y epel-release"
+}
+
+task "jq-install" {
+  check   = "yum list installed jq"
+  apply   = "yum makecache; yum install -y jq"
+  depends = ["task.epel-install"]
+}