diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..d7576eb71af --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +docker-db-data \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1328658c6ab..73316769e63 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ public/attachments public/export storage tmp + +# docker-compose database directory +docker-db-data diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 00000000000..74e5f57b6a4 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,97 @@ +# Using Docker and Docker Compose to run OpenStreetMap + +Using [Docker](https://www.docker.com/) will allow you to install the OpenStreetMap application and all its dependencies in Docker images and then run them in containers, almost with a single command. You will need to install Docker and Docker Compose on your development machine: + +- [Install Docker](https://docs.docker.com/install/) +- [Install Docker Compose](https://docs.docker.com/compose/install/) + +These instructions gloss over the precise details of the dependencies and their configuration but these can be found in full detail at [INSTALL.md](INSTALL.md). + +The first step is to fork/clone the repo to your local machine. The repository is reasonably large (~150MB) and it's unlikely that you need the full history. If you are happy to wait for it all to download, run: + + git clone https://github.com/openstreetmap/openstreetmap-website.git + +To clone only the most recent version (~23MB), instead use a 'shallow clone': + + git clone --depth=1 https://github.com/openstreetmap/openstreetmap-website.git + +Now change working directory to the `openstreetmap-website`: + + cd openstreetmap-website + +### Storage setup + + cp config/example.storage.yml config/storage.yml + +### Database + + cp config/docker.database.yml config/database.yml + +### App configuration + + cp config/settings.yml config/settings.local.yml + +### Installation + +In the root directory run: + + docker-compose build + +Now if this is your first time running or you have removed cache this will take some time to complete. So grab tea/coffee and sit tight. Once the Docker images have finished building you can launch the images as containers. + +To launch the app run: + + docker-compose up -d + +This will launch one Docker container for each 'service' specified in `docker-compose.yml` and run them in the background. There are two options for inspecting the logs of these running containers: + +- You can tail logs of a running container with a command like this: `docker-compose logs -f web` or `docker-compose logs -f db`. +- Instead of running the containers in the background with the `-d` flag, you can launch the containers in the foreground with `docker-compose up`. The downside of this is that the logs of all the 'services' defined in `docker-compose.yml` will be intermingled. If you don't want this you can mix and match - for example, you can run the database in background with `docker-compose up -d db` and then run the Rails app in the foreground via `docker-compose up web`. + +### Migrations + +While the `db' service is running, open another terminal windows and run: + + docker-compose run --rm web rake db:migrate + +### Node.js modules + +We use Yarn to manage the Node.js modules required for the project: + + docker-compose run --rm web rake yarn:install + +Once these are complete you should be able to visit the app at http://localhost:3000 + +If localhost does not work, you can use the IP address of the docker-machine. + +### Tests + + docker-compose run --rm web rake test:db + +### Bash + +If you want to get into a web container and run specific commands you can fire up a throwaway container to run bash in via: + + docker-compose run --rm web bash + +Alternatively, if you want to use the already-running `web` container then you can `exec` into it via: + + docker-compose exec web bash + +Similarly, if you want to `exec` in the db container use: + + docker-compose exec db bash + +### Populating the database + +This installation comes with no geographic data loaded. You can either create new data using one of the editors (Potlatch 2, iD, JOSM etc) or by loading an OSM extract. + +After installing but before creating any users or data, import an extract with [Osmosis](https://wiki.openstreetmap.org/wiki/Osmosis) and the `--write-apidb` task. The `web` container comes with `osmosis` pre-installed. So to populate data with a `.osm.pbf` use the following command: + + docker-compose run --rm web osmosis \ + --read-pbf /path/to/file.osm.pbf \ + --write-apidb \ + host="db" \ + database="openstreetmap" \ + user="openstreetmap" \ + validateSchemaVersion="no" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..9788341057e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +FROM ruby:2.5 + +# fixes dpkg man page softlink error while installing postgresql-client [source: https://stackoverflow.com/a/52655008/5350059] +RUN mkdir -p /usr/share/man/man1 && \ + mkdir -p /usr/share/man/man7 + +# npm is not available in Debian repo so following official instruction [source: https://github.com/nodesource/distributions/blob/master/README.md#debinstall] +RUN curl -sL https://deb.nodesource.com/setup_10.x -o nodesource_setup.sh && \ + bash nodesource_setup.sh && \ + rm -f nodesource_setup.sh + +# install packages +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + build-essential \ + curl \ + imagemagick \ + libarchive-dev \ + libffi-dev \ + libmagickwand-dev \ + libpq-dev \ + libsasl2-dev \ + libxml2-dev \ + libxslt1-dev \ + locales \ + nodejs \ + osmosis \ + postgresql-client \ + ruby-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install npm packages +RUN npm install -g --unsafe-perm \ + phantomjs-prebuilt \ + yarn + +# Setup app location +RUN mkdir -p /app +WORKDIR /app + +# Install gems +ADD Gemfile* /app/ +RUN bundle install + +# Setup local +RUN sed -i -e 's/# en_GB.UTF-8 UTF-8/en_GB.UTF-8 UTF-8/' /etc/locale.gen && \ + echo 'LANG="en_GB.UTF-8"'>/etc/default/locale && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_GB.UTF-8 + +ENV LANG en_GB.UTF-8 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..d2939d4e421 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +docker-build: + docker-compose build + +docker-up: + docker-compose up -d + +docker-db-migrate: + docker-compose run --rm web rake db:migrate + +docker-test: + docker-compose run --rm web rake test:db + +docker-populate-db: + wget https://download.geofabrik.de/north-america/us/district-of-columbia-latest.osm.pbf + docker-compose run --rm web osmosis \ + -verbose \ + --read-pbf district-of-columbia-latest.osm.pbf \ + --write-apidb \ + host="db" \ + database="openstreetmap" \ + user="openstreetmap" \ + validateSchemaVersion="no" diff --git a/config/docker.database.yml b/config/docker.database.yml new file mode 100644 index 00000000000..6e6e5fd9958 --- /dev/null +++ b/config/docker.database.yml @@ -0,0 +1,20 @@ +# This configuration is tailored for use with docker-compose. See DOCKER.md for more information. + +development: + adapter: postgresql + database: openstreetmap + username: openstreetmap + password: openstreetmap + host: db + encoding: utf8 + +# Warning: The database defined as 'test' will be erased and +# re-generated from your development database when you run 'rake'. +# Do not set this db to the same as development or production. +test: + adapter: postgresql + database: osm_test + username: postgres + password: + host: db + encoding: utf8 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000000..70730febfca --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3" + +services: + web: + build: + context: . + volumes: + - .:/app + ports: + - "3000:3000" + command: bundle exec rails s -p 3000 -b '0.0.0.0' + depends_on: + - db + + db: + build: + context: . + dockerfile: docker/postgres/Dockerfile + ports: + - "54321:5432" + environment: + POSTGRES_DB: openstreetmap + volumes: + - ./docker-db-data:/var/lib/postgresql/data diff --git a/docker/postgres/Dockerfile b/docker/postgres/Dockerfile new file mode 100644 index 00000000000..672e2e7a623 --- /dev/null +++ b/docker/postgres/Dockerfile @@ -0,0 +1,4 @@ +FROM postgres:11 + +# Add db init script to install OSM-specific Postgres functions/extensions. +ADD docker/postgres/openstreetmap-postgres-init.sh /docker-entrypoint-initdb.d/ diff --git a/docker/postgres/openstreetmap-postgres-init.sh b/docker/postgres/openstreetmap-postgres-init.sh new file mode 100755 index 00000000000..d8c619d45fd --- /dev/null +++ b/docker/postgres/openstreetmap-postgres-init.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex + +# Create 'openstreetmap' user +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" <<-EOSQL + CREATE USER openstreetmap PASSWORD 'openstreetmap'; + GRANT ALL PRIVILEGES ON DATABASE openstreetmap TO openstreetmap; +EOSQL + +# Create btree_gist extensions +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -c "CREATE EXTENSION btree_gist" openstreetmap