V1: Initial version

Replaced Django+MySQL with Tornado+ES

Author: marcelometal

.coveragerc

@@ -0,0 +1,16 @@
+[run]
+omit =
+    tests*
+branch = True
+source =
+    politicos_api
+    collector
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    from urllib.parse import parse_qs
+    except ImportError:

.editorconfig

@@ -0,0 +1,20 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true
+
+[Makefile]
+indent_style = tab
+indent_size = 4
+
+[*.html]
+indent_size = 2
+
+[*.css]
+indent_size = 2
+
+[*.json]
+indent_size = 2
+insert_final_newline = ignore

.gitignore

@@ -58,7 +58,6 @@ report.html
 # VIM
 *.swp
 
-# settings
-local.config
-
-logs
+# Coverage
+.coverage
+htmlcov/

.travis.yml

@@ -1,11 +1,19 @@
 language: python
 
+before_install:
+  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.4.2.deb
+  - sudo dpkg -i --force-confnew elasticsearch-6.4.2.deb
+  - sudo /etc/init.d/elasticsearch start
+  - until curl --silent -XGET --fail http://localhost:9200; do printf '.'; sleep 1; done
+  - sudo apt-get install redis-server
+
 python:
-    - "2.7"
+    - "3.6"
 
 env:
   matrix:
     - TEST_TYPE=lint
+    - TEST_TYPE=unit
 
 install:
     - make setup

AUTHORS

@@ -0,0 +1 @@
+Marcelo Jorge Vieira

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.7
+
+RUN	apt-get update && \
+	apt-get -y install libcurl4-openssl-dev build-essential libssl-dev
+
+RUN mkdir -p /usr/share/politicos-api
+WORKDIR /usr/share/politicos-api
+
+COPY . /usr/share/politicos-api
+RUN pip install --no-cache-dir -r /usr/share/politicos-api/requirements.txt

Makefile

@@ -1,28 +1,32 @@
-setup:
-	@pip install -U --process-dependency-links  -e .\[tests\]
-
 run:
-	@python manage.py runserver 0.0.0.0:8000
-
-data:
-	@python manage.py migrate
-
-initial_data:
-	@python manage.py countries
-	@python manage.py states
-	@python manage.py ethnicity
-	@python manage.py education
-	@python manage.py political_party
-	@python manage.py mandate_event_type
-	@python manage.py political_office
-	@python manage.py institutions
-	@python manage.py elections
+	@python server.py
 
-clean_pycs:
-	@find . -name "*.pyc" -delete
+setup:
+	@pip install -r requirements.txt -r requirements_dev.txt
+
+setup-prod:
+	@pip install -r requirements.txt
 
 lint:
 	@flake8
 
+test: coverage lint
+
+unit:
+	@coverage run --branch `which nosetests` -vv --with-yanc -s tests
 
-.PHONY: initial_data clean_pycs setup run
+focus:
+	@coverage run --branch `which nosetests` -vv --with-yanc --with-focus -s tests
+
+coverage: unit
+	@coverage report -m
+
+coverage-html: coverage
+	@coverage html
+
+collect:
+	@python collector.py
+
+clean_pycs:
+	@find . -name "*.pyc" -delete
+	@find . -type d -name __pycache__ -delete

README.md

@@ -1,52 +1,102 @@
-# Politicos API [![Build Status](https://secure.travis-ci.org/olhoneles/politicos.png?branch=master)](https://travis-ci.org/olhoneles/politicos)
+# Politicos API [![Build Status][build-status]][travis]
 
-Aqui você vai encontrar os dados, ferramentas e recursos para realizar pesquisas, desenvolver aplicativos, visualizações de dados, e muito mais.
+Aqui você vai encontrar os dados, ferramentas e recursos para realizar
+pesquisas, desenvolver aplicativos, visualizações de dados e muito mais.
 
 
 ## Instalação
 
-1.  Clone o repositório:
+Temos duas formas para rodar o projeto, via [docker compose][docker-compose] ou
+instalando as dependências manualmente.
 
-        git clone https://github.com/olhoneles/politicos.git
+### Clone o repositório:
 
-1.  Crie um [*virtualenv*](http://virtualenvwrapper.readthedocs.org/en/latest/install.html):
+```
+git clone https://github.com/olhoneles/politicos.git
+```
 
-        cd politicos
-        mkvirtualenv politicos
+### Docker Compose
 
-1.  Instale as dependências:
+#### Subindo o projeto
 
-        make setup
+```
+cd politicos
+docker-compose up
+```
 
-1.  Crie o banco de dados:
+#### Para coletar os dados do TSE:
 
-        make data
+```
+make collect
+```
 
-1.  Inicialize o projeto:
+#### Para visualizar os dados:
 
-        make run
+Abra seu navegador e entre no endereço http://localhost:8888
 
-1.  Se você quiser sobrescrever algumas configurações do settings.py, como SECRET_KEY, DATABASES, ALLOWED_HOSTS, por favor crie o arquivo `politicos/local.config`.
+### Manualmente
 
+#### Crie um [*virtualenv*][virtualenv]:
+
+```
+cd politicos
+mkvirtualenv politicos
+```
+
+#### Instale as dependências (python):
+
+```
+make setup
+```
+
+#### Instale o ElasticSearch e o Redis
+
+
+* [ElasticSearch][elasticsearch]
+* [Redis][redis-server]
+
+#### Para coletar os dados do TSE:
+
+```
+make collect
+```
+
+#### Inicialize o projeto:
+
+```
+make run
+```
+
+Abra seu navegador e entre no endereço http://localhost:8888
 
 ## Como Contribuir
 
-Participe da [lista de email](http://listas.olhoneles.org/cgi-bin/mailman/listinfo/montanha-dev) e também via IRC no canal [#olhoneles](irc://irc.freenode.net:6667/olhoneles) na Freenode.
+Participe da [lista de email][lista] e também via IRC no canal
+[#olhoneles][freenode] na Freenode.
 
 Faça uma cópia do repositório e envie seus pull-requests.
 
 
 ## Licença
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+details.
+
+You should have received a copy of the GNU Affero General Public License along
+with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+[virtualenv]: http://virtualenvwrapper.readthedocs.org/en/latest/install.html
+[lista]: http://listas.olhoneles.org/cgi-bin/mailman/listinfo/montanha-dev
+[freenode]: irc://irc.freenode.net:6667/olhoneles
+[build-status]: https://secure.travis-ci.org/olhoneles/politicos.png?branch=master
+[travis]: https://travis-ci.org/olhoneles/politicos
+[docker-compose]: https://docs.docker.com/compose/install/
+[elasticsearch]: https://www.elastic.co/downloads/elasticsearch
+[redis-server]: https://redis.io/download

apache.example.conf

@@ -0,0 +1,43 @@
+# a2enmod lbmethod_byrequests proxy proxy_balancer proxy_http ssl
+
+<VirtualHost *:80>
+        ProxyRequests off
+        ServerName politicosapi.localdomain
+
+        <Proxy balancer://mycluster>
+                BalancerMember http://localhost:8000
+                BalancerMember http://localhost:8001
+                BalancerMember http://localhost:8002
+                BalancerMember http://localhost:8003
+
+                # Security "technically we aren't blocking anyone but this is
+                # the place to make those changes.
+                Require all granted
+                # In this example all requests are allowed.
+
+                # Load Balancer Settings
+                # We will be configuring a simple Round Robin style load
+                # balancer.  This means that all webheads take an equal share
+                # of of the load.
+                ProxySet lbmethod=byrequests
+
+        </Proxy>
+
+        # balancer-manager
+        # This tool is built into the mod_proxy_balancer module and will allow
+        # you to do some simple modifications to the balanced group via a gui
+        # web interface.
+        <Location /balancer-manager>
+                SetHandler balancer-manager
+                # I recommend locking this one down to your your office
+                Require host politicosapi.localdomain
+
+        </Location>
+
+        # Point of Balance
+        # This setting will allow to explicitly name the the location in the
+        # site that we want to be balanced, in this example we will balance "/"
+        # or everything in the site.
+        ProxyPass /balancer-manager !
+        ProxyPass / balancer://mycluster/
+</VirtualHost>

collector.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2018, Marcelo Jorge Vieira <[email protected]>
+#
+#  This program is free software: you can redistribute it and/or modify it
+#  under the terms of the GNU Affero General Public License as published by the
+#  Free Software Foundation, either version 3 of the License, or (at your
+#  option) any later version.
+#
+#  This program is distributed in the hope that it will be useful, but WITHOUT
+#  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
+#  for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+
+from collector.tse import TSE
+from collector.tse_headers import year_headers
+
+
+# FIXME
+FILES_DIR = '/home/metal/Downloads/blah'
+
+
+def main():
+    FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    logging.basicConfig(format=FORMAT, level=logging.INFO)
+    # mute elasticsearch INFO logs
+    log = logging.getLogger('elasticsearch')
+    log.setLevel('ERROR')
+
+    for year in year_headers.keys():
+        tse = TSE(year, path=FILES_DIR)
+        tse.download_and_extract(remove_tmp_dir=False, remove_zip=False)
+        tse.all_candidates()
+
+
+if __name__ == '__main__':
+    main()