initial commit

.gitignore

@@ -0,0 +1,3 @@
+# Makefile
+#env_make
+

Dockerfile

@@ -0,0 +1,72 @@
+FROM centos:centos7.2.1511
+LABEL maintainer "https://github.com/dmartin35"
+LABEL name "ReadTheDocs"
+
+RUN yum -y update
+RUN yum -y install python-setuptools
+RUN easy_install supervisor
+# install gettext for envsubst cmd
+RUN yum -y install gettext
+RUN yum -y install psmisc
+
+# exec main install scripts
+ADD install /tmp/install
+RUN chmod +x /tmp/install/*.sh
+RUN /bin/bash /tmp/install/deps.sh
+RUN /bin/bash /tmp/install/postgresql.sh
+RUN /bin/bash /tmp/install/nginx.sh
+RUN /bin/bash /tmp/install/elastic.sh
+RUN /bin/bash /tmp/install/redis.sh
+RUN /bin/bash /tmp/install/latex.sh
+
+# RTD - source & venv & configure
+RUN /bin/bash /tmp/install/rtd.sh
+RUN mkdir -p /opt/readthedocs.org/ini
+COPY resources/rtd/conf.py.tmpl.patch /opt/readthedocs.org/ini/conf.py.tmpl.patch
+RUN cd /opt/readthedocs.org && patch -p1 < /opt/readthedocs.org/ini/conf.py.tmpl.patch
+COPY resources/rtd/settings/prod.py /opt/readthedocs.org/readthedocs/settings/prod.py
+RUN chmod 644 /opt/readthedocs.org/readthedocs/settings/prod.py
+# apply patch - fixing https://github.com/rtfd/readthedocs.org/issues/2261 for firefox
+COPY resources/rtd/details.css.patch /opt/readthedocs.org/ini/details.css.patch
+RUN cd /opt/readthedocs.org && patch -p1 < /opt/readthedocs.org/ini/details.css.patch
+
+#LOGROTATE
+COPY resources/logrotate.d/celery /etc/logrotate.d/celery
+COPY resources/logrotate.d/uwsgi /etc/logrotate.d/uwsgi
+COPY resources/logrotate.d/elasticsearch /etc/logrotate.d/elasticsearch
+COPY resources/logrotate.d/nginx /etc/logrotate.d/nginx
+COPY resources/logrotate.d/redis /etc/logrotate.d/redis
+COPY resources/logrotate.d/rtd /etc/logrotate.d/rtd
+
+#CELERY
+RUN mkdir -p /etc/conf.d/
+COPY resources/celery.conf /etc/conf.d/celery
+RUN mkdir -p /var/log/celery
+RUN mkdir -p /var/run/celery
+
+#UWSGI
+COPY resources/uwsgi.ini /opt/readthedocs.org/ini/uwsgi.ini
+RUN mkdir -p /var/log/uwsgi/
+RUN mkdir -p /etc/uwsgi/vassals/
+RUN ln -s /opt/readthedocs.org/ini/uwsgi.ini /etc/uwsgi/vassals/
+
+#NGINX
+RUN mkdir /run/nginx && chown nginx:nginx /run/nginx
+COPY resources/nginx_default.conf /etc/nginx/nginx.conf
+COPY resources/nginx_rtd.conf /opt/readthedocs.org/ini/nginx.conf.tpl
+RUN ln -s /opt/readthedocs.org/ini/nginx.conf /etc/nginx/conf.d/
+
+#ELASTICSEARCH
+COPY resources/configure_elasticsearch.py /tmp/configure_elasticsearch.py
+
+# configure script
+COPY configure.sh /tmp/configure.sh
+RUN chmod +x /tmp/configure.sh
+
+EXPOSE 80 9001
+
+COPY supervisord.conf /etc/supervisord.conf
+ADD entry_point.sh /entry_point.sh
+RUN chmod -v +x /entry_point.sh
+CMD ["/entry_point.sh"]
+

Makefile

@@ -0,0 +1,35 @@
+include env_make
+NS = dmartin35
+VERSION ?= latest
+
+REPO = rtd
+NAME = rtd
+INSTANCE = default
+
+.PHONY: build push shell run start stop rm release
+
+build:
+	docker build -t $(NS)/$(REPO):$(VERSION) .
+
+push:
+	docker push $(NS)/$(REPO):$(VERSION)
+
+shell:
+	docker run --rm --name $(NAME)-$(INSTANCE) -i -t $(PORTS) $(VOLUMES) $(ENV) $(NS)/$(REPO):$(VERSION) /bin/bash
+
+run:
+	docker run --rm --name $(NAME)-$(INSTANCE) $(PORTS) $(VOLUMES) $(ENV) $(NS)/$(REPO):$(VERSION)
+
+start:
+	docker run -d --name $(NAME)-$(INSTANCE) $(PORTS) $(VOLUMES) $(ENV) $(NS)/$(REPO):$(VERSION)
+
+stop:
+	docker stop $(NAME)-$(INSTANCE)
+
+rm:
+	docker rm $(NAME)-$(INSTANCE)
+
+release: build
+	make push -e VERSION=$(VERSION)
+
+default: build

README.md

@@ -1,2 +1,33 @@
-# docker-readthedocs
+# readthedocs-docker
 Project to run ReadTheDocs server inside a docker container
+
+
+Build the image
+---------------
+
+To create the docker image, execute the following command ``make``
+
+Run the server
+--------------
+
+To run the docker image and launch the RTD web server, execute ``make run`` or ``make  start`` (to run in detached mode)
+
+.. note:: docker options are indicated in the env_make file
+
+.. warning:: no data (docs/artefacts) are persistent when stopping the docker container. Volumes must be used to do so.
+
+
+Persistent Data Storage
+-----------------------
+
+@todo - run with volumes on host, for database, docs & artefacts to be stored on host for persistent data storage
+
+
+Limitations
+-----------
+
+- only supports svn, git and mercurial
+- Email verification on signup has been turned off
+- A default admin user is automatically created to be used for slumber API
+- subdomains are not tested (not sure it still works after patching doc with full absolute url)
+- [bug] links are broken in doc's search result page (contains <filename>.<ext>.html that cannot be found)

configure.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+
+
+function configure_postgreSQL(){
+    psql -U postgres -c "CREATE USER rtd WITH PASSWORD 'rtd'; ALTER USER rtd WITH SUPERUSER;"
+    createdb -U postgres --owner rtd readthedoc
+}
+
+
+
+function configure_django(){
+    # create secret_key file
+    cat /dev/urandom | tr -dc 'a-zA-Z0-9!#$%^&*(-_=+)' | fold -w 50 | head -n 1  > /opt/readthedocs.org/ini/secret_key.txt
+
+}
+
+function django_admin() {
+    # run admin command
+    source /opt/venv/rtd/bin/activate
+    export DJANGO_SETTINGS_MODULE=readthedocs.settings.prod
+    cd /opt/readthedocs.org/
+    python manage.py makemigrations
+    python manage.py migrate
+    echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', '[email protected]', 'azerty1234')" | python manage.py shell
+    python manage.py collectstatic --noinput
+    deactivate
+}
+
+
+function configure_elesticsearch(){
+
+    source /opt/venv/rtd/bin/activate
+    export PYTHONPATH=/opt/readthedocs.org/
+    export DJANGO_SETTINGS_MODULE=readthedocs.settings.prod
+    python /tmp/configure_elasticsearch.py
+    deactivate
+
+}
+
+
+function configure_rtd_folders(){
+
+    for folder in media private_cname_project private_cname_root private_web_root public_cname_project public_cname_root public_web_root user_builds
+    do
+        mkdir -p /opt/readthedocs.org/$folder
+    done
+
+}
+
+
+configure_postgreSQL
+configure_django
+django_admin
+configure_elesticsearch
+configure_rtd_folders
+

entry_point.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [ -z "$SERVER" ]; then
+    export SERVER='readthedocs.com'
+fi
+
+ALREADY_CONFIGURED=`ls /opt/readthedocs.org/ini | grep configured.ok`
+if [ -z $ALREADY_CONFIGURED ]; then
+
+    source /etc/sysconfig/elasticsearch && /usr/share/elasticsearch/bin/elasticsearch -d -p /var/run/elasticsearch/elasticsearch.pid -Des.default.config=$CONF_FILE -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.work=$WORK_DIR -Des.default.path.conf=$CONF_DIR
+
+    su - postgres -c '/usr/bin/pg_ctl start -D /var/lib/pgsql/data -s -o "-p 5432" -w -t 300'
+    /tmp/configure.sh
+    su - postgres -c '/usr/bin/pg_ctl stop -D /var/lib/pgsql/data -s -o "-p 5432" -w -t 300'
+
+    kill -9 `pgrep -f elasticsearch`
+    rm -rf /var/run/elasticsearch/elasticsearch.pid
+    chown -R elasticsearch:elasticsearch /var/lib/elasticsearch/
+
+    echo 'OK' > /opt/readthedocs.org/ini/configured.ok
+
+fi
+
+# replaces ${SERVER} with host IP
+envsubst < /opt/readthedocs.org/ini/nginx.conf.tpl > /opt/readthedocs.org/ini/nginx.conf
+
+exec /usr/bin/supervisord

env_make

@@ -0,0 +1,9 @@
+PORTS = \
+    -p 80:80 \
+    -p 9001:9001
+
+VOLUMES =
+# -v /var/log/docker:/var/log
+
+ENV = \
+  -e SERVER=127.0.0.1

install/deps.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+yum update -y
+yum install -y gcc
+yum install -y wget patch
+yum install -y svn git mercurial
+yum install -y libxml2-devel libxslt-devel
+yum install -y python-devel policycoreutils-python
+yum install -y openldap-devel
+
+

install/elastic.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cd /tmp
+yum install -y java
+wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.5.2.noarch.rpm
+rpm -ivh elasticsearch-1.5.2.noarch.rpm
+/usr/share/elasticsearch/bin/plugin -install elasticsearch/elasticsearch-analysis-icu/2.5.0

install/latex.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# http://pj.freefaculty.org/EL/7/x86_64/
+yum install -y wget
+
+wget http://pj.freefaculty.org/EL/7/x86_64/texlive-capt-of-svn29803.0-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-needspace-svn29601.1.3d-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-eqparbox-svn29419.4.0-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-environ-svn29600.0.3-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-trimspaces-svn15878.1.1-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-environ-svn29600.0.3-1.1EL7.noarch.rpm http://pj.freefaculty.org/EL/7/x86_64/texlive-eqparbox-svn29419.4.0-1.1EL7.noarch.rpm
+wget http://pj.freefaculty.org/EL/7/x86_64/texlive-upquote-svn26059.v1.3-1.1EL7.noarch.rpm
+wget http://pj.freefaculty.org/EL/7/x86_64/texlive-tabulary-doc-svn15878.0.9-1.1EL7.noarch.rpm
+wget http://pj.freefaculty.org/EL/7/x86_64/texlive-tabulary-svn15878.0.9-1.1EL7.noarch.rpm
+
+yum install -y texlive-pdftex texlive-tetex texlive-latex-bin texlive-makeindex texlive-texconfig texlive-cmap texlive-metafont texlive-fontspec texlive-ec texlive-cm texlive-fancyhdr texlive-fancybox texlive-titlesec texlive-framed texlive-pdftex-def texlive-threeparttable texlive-mdwtools texlive-wrapfig texlive-parskip
+yum install -y texlive-pdftex-doc texlive-pdftex texlive-caption
+yum install -y texlive-times texlive-helvetic texlive-courier texlive-dvips texlive-misc texlive-multirow
+yum install -y texlive-fncychap texlive-ifluatex
+
+rpm -i texlive-upquote-svn26059.v1.3-1.1EL7.noarch.rpm
+rpm -i texlive-capt-of-svn29803.0-1.1EL7.noarch.rpm
+rpm -i texlive-needspace-svn29601.1.3d-1.1EL7.noarch.rpm
+rpm -i texlive-eqparbox-svn29419.4.0-1.1EL7.noarch.rpm
+rpm -i texlive-environ-svn29600.0.3-1.1EL7.noarch.rpm
+rpm -i texlive-trimspaces-svn15878.1.1-1.1EL7.noarch.rpm
+rpm -i texlive-environ-svn29600.0.3-1.1EL7.noarch.rpm
+rpm -i texlive-eqparbox-svn29419.4.0-1.1EL7.noarch.rpm
+rpm -i texlive-tabulary-svn15878.0.9-1.1EL7.noarch.rpm
+rpm -i texlive-tabulary-doc-svn15878.0.9-1.1EL7.noarch.rpm

install/nginx.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+yum install -y epel-release
+yum install -y nginx

install/postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+yum install -y postgresql-devel
+yum install -y  postgresql postgresql-server postgresql-contrib
+su - postgres -c "initdb --auth-host=md5"

install/redis.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+yum install -y redis

install/rtd.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+function create_venv(){
+    cd /tmp
+    curl -O https://bootstrap.pypa.io/get-pip.py
+    python get-pip.py
+    pip install virtualenv
+    /usr/bin/virtualenv /opt/venv/rtd
+    source /opt/venv/rtd/bin/activate
+    pip install -r /opt/readthedocs.org/requirements.txt
+    pip install uwsgi
+    pip install psycopg2
+    deactivate
+}
+
+function get_RTD_src() {
+    cd /opt/
+    git clone https://github.com/rtfd/readthedocs.org.git
+    cd /opt/readthedocs.org/
+    git checkout 340ea0e55d185aad2a86a76bd81a89ce8be13d51
+}
+
+get_RTD_src
+create_venv

resources/celery.conf

@@ -0,0 +1,7 @@
+CELERYD_OPTS="--concurrency 8 -Q celery,web"
+CELERY_BIN="/opt/venv/rtd/bin/python /opt/readthedocs.org/manage.py celeryd"
+CELERYD_PID_FILE="/var/run/celery/rtd-worker.pid"
+CELERYD_LOG_FILE="/var/log/celery/rtd-worker.log"
+CELERYD_LOG_LEVEL="INFO"
+
+DJANGO_SETTINGS_MODULE="readthedocs.settings.prod"

resources/configure_elasticsearch.py

@@ -0,0 +1,15 @@
+from readthedocs.search.indexes import Index, PageIndex, ProjectIndex, SectionIndex
+#from search.indexes import Index, PageIndex, ProjectIndex, SectionIndex
+
+# Create the index.
+index = Index()
+index_name = index.timestamped_index()
+index.create_index(index_name)
+index.update_aliases(index_name)
+# Update mapping
+proj = ProjectIndex()
+proj.put_mapping()
+page = PageIndex()
+page.put_mapping()
+sec = SectionIndex()
+sec.put_mapping()

resources/logrotate.d/celery

@@ -0,0 +1,10 @@
+/var/log/celery/*.log {
+    missingok
+    rotate 10
+    maxage 7
+    compress
+    nodelaycompress
+    daily
+    notifempty
+    copytruncate
+}

resources/logrotate.d/elasticsearch

@@ -0,0 +1,10 @@
+/var/log/elasticsearch/*.log {
+    missingok
+    rotate 10
+    maxage 7
+    compress
+    nodelaycompress
+    daily
+    notifempty
+    copytruncate
+}

resources/logrotate.d/nginx

@@ -0,0 +1,10 @@
+/var/log/nginx/*.log {
+    missingok
+    rotate 10
+    maxage 7
+    compress
+    nodelaycompress
+    daily
+    notifempty
+    copytruncate
+}