Secrets for local development

Try to retrieve secrets for local development.

Reference: CV2-2687.

Author: caiosba

.travis.yml

@@ -11,19 +11,16 @@ before_script:
   - export FALLBACK_BRANCH=$([ "$TRAVIS_BRANCH" == "master" ] && echo "main" || echo "develop")
   - git submodule foreach 'bash -c "git checkout develop ; git checkout $FALLBACK_BRANCH ; git checkout $TRAVIS_BRANCH ; exit 0"'
   - git submodule foreach git rev-parse --abbrev-ref HEAD
-  # A couple fixes for Fetch
-  - sed -i '/0.0.0/ i \'"${TAB}"'\until curl --silent -XGET --fail http://elasticsearch:9200; do printf \.\; sleep 1; done' fetch/Makefile
-  - sed -i '/-c 5/ i \'"${TAB}"'\until curl --silent -XGET --fail http://elasticsearch:9200; do printf \.\; sleep 1; done' fetch/Makefile
   - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 script:
   - ./bin/first-build.sh
   - until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
   - sleep 120
-  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3200)!= "200" ));  then exit 1; fi;
-  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3100)!= "200" ));  then exit 1; fi;
-  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3333)!= "200" ));  then exit 1; fi;
-  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000)!= "200" ));  then exit 1; fi;
-  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:8001)!= "200" ));  then exit 1; fi;
+  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3200)!= "200" ));  then exit 1; fi; # Pender
+  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3100)!= "200" ));  then exit 1; fi; # Alegre
+  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3333)!= "200" ));  then exit 1; fi; # Check Web
+  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:3000)!= "200" ));  then exit 1; fi; # Check API
+  - if (( $(curl -s -o /dev/null -w "%{http_code}" http://localhost:8001)!= "200" ));  then exit 1; fi; # Check Search
   - docker ps --format '{{.Image}}' | sort >> docker_up_output.txt
   - diff docker_up_output.txt test/image_names.txt
 notifications:

README.md

@@ -23,9 +23,9 @@ This is a [Docker Compose](https://docs.docker.com/compose/) configuration that
 - Click "I already have an account" and login using your credentials
 - Enjoy Check! :tada:
 
-**Note:** As you run the applications you may need credentials and configuration that are not provided by copying `.example` files during the initial build. You have to add the necessary credentials in the config files to use some features correctly:
-e.g: the [FACEBOOK APP ID](https://github.com/meedan/pender/blob/develop/config/config.yml.example#L64) to get Facebook social metrics and to run the Facebook related tests
+**Note 1:** For security reasons, not all credentials and configuration values are provided by copying `.example` files during the initial build. For Meedan members, you need to set your `AWS_PROFILE` environment variable, login to AWS (`aws sso login`) and then the script `bin/first-build.sh` will retrieve and set the required values for you. If you're not a Meedan member, you need to set at least the `google_client_id` and `google_client_secret` values in `check-api/config/config.yml`, [here is how you can get those](https://developers.google.com/identity/protocols/oauth2). Other optional features can be enabled by setting the required credentials, for example, the [FACEBOOK APP ID](https://github.com/meedan/pender/blob/develop/config/config.yml.example#L64) is needed to get Facebook social metrics and to run the Facebook related tests.
 
+**Note 2:** For performance reasons, some services (that are not needed to run the application with its basic functionality) are disabled by default (e.g., commented in the Docker Compose file). If you need those services, please uncomment them in `docker-compose.yml`. If you may need to increase the amount of memory allocated for Docker in order for it to work.
 
 ## Available services and container names
 

alegre

@@ -1,5 +1,22 @@
 #!/bin/bash
+
+# Go to the develop branch of each repository
 git submodule foreach bash -c 'git checkout develop'
+
+# Copy the example files
 find . -name '*.example' -not -path '*apollo*' | while read f; do cp "$f" "${f%%.example}"; done
+
+# Replace secrets (if you have access, login first with "aws sso login")
+replace_secret () {
+  app=$1
+  file=$2
+  key=$3
+  value=$(aws ssm get-parameter --region eu-west-1 --name "/local/$app/$key" | grep Value | sed 's/.*"Value": "\(.*\)",/\1/g')
+  sed -i "s/$key: 'SECRET'/$key: '$value'/g" "$app/$file"
+}
+replace_secret 'check-api' 'config/config.yml' 'google_client_id'
+replace_secret 'check-api' 'config/config.yml' 'google_client_secret'
+
+# Build & Run
 docker-compose build
 docker-compose up --abort-on-container-exit

bin/first-build.sh

@@ -301,38 +301,38 @@ services:
       DEPLOY_ENV: local
     networks:
       - dev
-  fetch:
-    volumes:
-      - "./fetch:/app"
-    build: fetch
-    platform: linux/x86_64
-    depends_on:
-      - elasticsearch
-      - redis
-      - alegre
-      - pender
-    ports:
-      - "9292:9292"
-    env_file:
-      - fetch/.env_file
-    command: start_server
-    networks:
-      - dev
-  fetch-background:
-    volumes:
-      - "./fetch:/app"
-    platform: linux/x86_64
-    depends_on:
-      - elasticsearch
-      - redis
-      - alegre
-      - pender
-    build: fetch
-    env_file:
-      - fetch/.env_file
-    command: run_worker
-    networks:
-      - dev
+#  fetch:
+#    volumes:
+#      - "./fetch:/app"
+#    build: fetch
+#    platform: linux/x86_64
+#    depends_on:
+#      - elasticsearch
+#      - redis
+#      - alegre
+#      - pender
+#    ports:
+#      - "9292:9292"
+#    env_file:
+#      - fetch/.env_file
+#    command: start_server
+#    networks:
+#      - dev
+#  fetch-background:
+#    volumes:
+#      - "./fetch:/app"
+#    platform: linux/x86_64
+#    depends_on:
+#      - elasticsearch
+#      - redis
+#      - alegre
+#      - pender
+#    build: fetch
+#    env_file:
+#      - fetch/.env_file
+#    command: run_worker
+#    networks:
+#      - dev
   mark:
     build: check-mark
     platform: linux/x86_64

check-api

@@ -4,8 +4,6 @@ check_api-background
 check_bots
 check_check-slack-bot
 check_elasticsearch
-check_fetch
-check_fetch-background
 check_mark
 check_narcissus
 check_pender