Merge pull request #4347 from rmol/tbb-4290

Add TBB to functional tests

.circleci/config.yml

@@ -79,6 +79,7 @@ jobs:
 
       - run:
           name: Run tests
+          no_output_timeout: 20m
           command: |
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
             docker rm -f securedrop-test-xenial-py2 || true

.gitignore

@@ -8,6 +8,9 @@ install_files/ansible-base/group_vars/all/site-specific
 # ignore securedrop-app-code wheelhouse archive
 wheelhouse
 
+# ignore the instance information JSON file to prevent commit of private info
+securedrop/tests/functional/instance_information.json
+
 # ignore the ATHS/THS hostname file ansible places
 app-ssh-aths
 app-document-aths # leave this here for historic reasons
@@ -156,3 +159,6 @@ raw-test-output/
 #These files are used in CI/dev env for determining whether to rebuild docker
 #images
 *.checksum
+#Functional test logs
+securedrop/tests/functional/firefox.log
+securedrop/geckodriver.log

Makefile

@@ -26,7 +26,7 @@ install-mypy: ## pip install mypy in a dedicated python3 virtualenv
 	if [[ ! -d .python3/.venv ]] ; then \
 	  virtualenv --python=python3 .python3/.venv && \
 	  .python3/.venv/bin/pip3 install mypy ; \
-        fi
+	fi
 
 .PHONY: typelint
 typelint: install-mypy ## Runs type linting

admin/requirements-dev.in

@@ -4,7 +4,7 @@ flake8
 flaky
 mock
 pbr
-pip-tools>2.0.0
+pip-tools>=3.5.0,<4
 pylint
 pytest
 requests

admin/requirements-dev.txt

@@ -72,10 +72,6 @@ enum34==1.1.6 \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
     --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
     # via astroid, flake8
-first==2.0.1 \
-    --hash=sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e \
-    --hash=sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e \
-    # via pip-tools
 flake8==3.5.0 \
     --hash=sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0 \
     --hash=sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37
@@ -138,9 +134,9 @@ pbr==3.1.1 \
 pexpect==4.5.0 \
     --hash=sha256:9783f4644a3ef8528a6f20374eeb434431a650c797ca6d8df0d81e30fffdfa24 \
     --hash=sha256:9f8eb3277716a01faafaba553d629d3d60a1a624c7cf45daa600d2148c30020c
-pip-tools==2.0.2 \
-    --hash=sha256:90bbe6731a6a34d339bf14d90cf2892475386c7d06c458208191ac9992110e0a \
-    --hash=sha256:f11fc3bf1d87a0b4a68d4d595f619814e2396e92d75d7bdd2500edbf002ea6de
+pip-tools==3.5.0 \
+    --hash=sha256:0018485119986aebef136470c51bde85da736732079c687ab1d4c5eb5237e694 \
+    --hash=sha256:a395ca8bb32bcaea58c8da89a2518793d88b43b15152217ba117c4170e507af9
 pluggy==0.6.0 \
     --hash=sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff \
     --hash=sha256:d345c8fe681115900d6da8d048ba67c25df42973bda370783cd58826442dcd7c \

devops/gce-nested/gce-runner.sh

@@ -36,7 +36,6 @@ function fetch_junit_test_results() {
 # Copy up securedrop repo to remote server
 function copy_securedrop_repo() {
   rsync -a -e "ssh ${SSH_OPTS[*]}" \
-      --exclude .git \
       --exclude admin/.tox \
       --exclude '*.box' \
       --exclude '*.deb' \

devops/scripts/vnc-docker-connect.sh

@@ -9,17 +9,19 @@ set -e
 # Bomb out if container not running
 docker inspect securedrop-dev >/dev/null 2>&1 || (echo "ERROR: SD container not running."; exit 1)
 
+VNCPORT=5909
+
 # Maybe we are running macOS
 if [ "$(uname -s)" == "Darwin" ]; then
-    open "vnc://${USER}:[email protected]:5901" &
+    open "vnc://${USER}:[email protected]:${VNCPORT}" &
     exit 0
 fi
 
 # Find our securedrop docker ip
 SD_DOCKER_IP="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' securedrop-dev)"
 
 # Quit if the VNC port not found
-nc -w5 -z "$SD_DOCKER_IP" 5901 || (echo "ERROR: VNC server not found"; exit 1)
+nc -w5 -z "$SD_DOCKER_IP" ${VNCPORT} || (echo "ERROR: VNC server not found"; exit 1)
 
 if [ ! "$(which remote-viewer)" ]
 then
@@ -32,6 +34,8 @@ then
     exit 1
 fi
 
-rv_config="${TMPDIR:-/tmp}/func-vnc.ini"
-echo -e "[virt-viewer]\ntype=vnc\nhost=${SD_DOCKER_IP}\nport=5901\npassword=freedom" > "${rv_config}"
+
+rv_config="${TMPDIR:-/tmp}/sd-vnc.ini"
+echo -e "[virt-viewer]\ntype=vnc\nhost=${SD_DOCKER_IP}\nport=${VNCPORT}\npassword=freedom" > "${rv_config}"
+
 remote-viewer "${rv_config}" 2>/dev/null &

docs/set_up_admin_tails.rst

@@ -21,7 +21,7 @@ specific Tails session and click *Add*. And finally click *Start
 Tails*.
 
 .. note:: The *Administration password* is a one-time password. It
-	  will reset every time you shut down Tails.
+      will reset every time you shut down Tails.
 
 After Tails finishes booting, make sure you're connected to the Internet
 |Network| and that the Tor status onion icon is not crossed out
@@ -51,7 +51,7 @@ Start by running the following commands to download the git repository.
     git clone https://github.com/freedomofpress/securedrop.git
 
 .. note:: Since the repository is fairly large and Tor can be slow,
-	  this may take a few minutes.
+      this may take a few minutes.
 
 .. caution:: Do not download SecureDrop Git repository as a Zip file,
              or any other means. Only download by using the given git
@@ -151,8 +151,8 @@ To use the template:
    on the encrypted persistent volume, this additional passphrase is not necessary.
 
 .. warning:: You will not be able to access your passwords if you
-	     forget the master password or the location of the key
-	     file used to protect the database.
+         forget the master password or the location of the key
+         file used to protect the database.
 
 In case you wish to manually create a database, the suggested password fields in
 the admin template are:

docs/upgrade/xenial_prep.rst

@@ -53,7 +53,7 @@ from the command line on the *Application Server* by running the command:
 
 
 SecureDrop servers are updated automatically with the latest release version
-(0.12.2). Recently, some long-running SecureDrop instances were affected by a 
+(0.12.2). Recently, some long-running SecureDrop instances were affected by a
 bug which will cause any updates after 0.10.0 to fail. If your instance is still
 running 0.10.0, please `consult our advisory <https://securedrop.org/news/advisory-automatic-update-failure-version-0100-0110-some-securedrop-instances/>`_
 to update to the latest version.

install_files/ansible-base/roles/app-test/defaults/main.yml

@@ -2,3 +2,11 @@
 # Username for Apache service, used to set permissions on the
 # Source Interface config to enable logging in the staging environment.
 apache_user: www-data
+
+# Specify TBB functional_test.json file defaults
+tbb_timeout: 160
+tbb_selenium_user: journalist
+tbb_selenium_password: correct horse battery staple profanity oil chewy
+tbb_selenium_totp_secret: JHCOGO7VCER3EJ4L
+test_sd_root_dir: "{{ lookup('pipe','git rev-parse --show-toplevel') }}"
+tbb_funcfolder: "{{ test_sd_root_dir }}/securedrop/tests/functional/instance_information.json"

install_files/ansible-base/roles/app-test/files/tor_app.fact

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+#
+#
+# Ansible local fact producer for tor onion details of app server
+# to be chucked into /etc/ansible/facts.d/ with a .fact suffix
+
+import json
+import os
+
+TOR_SVC_PATH = "/var/lib/tor/services/"
+JOURNALIST_DIR = "journalist"
+SOURCE_DIR = "source"
+
+
+def extract_sd_onion_data(tor_path=TOR_SVC_PATH,
+                          journalist_dir=JOURNALIST_DIR,
+                          source_dir=SOURCE_DIR,
+                          protocol="http"):
+
+    for interface in [journalist_dir, source_dir]:
+        hostname_file = os.path.join(TOR_SVC_PATH, interface, 'hostname')
+        try:
+            with open(hostname_file) as f:
+                tor_config = f.readline().rstrip().split()
+
+            onion_addr = "{p}://{a}".format(p=protocol, a=tor_config[0])
+            auth_token = tor_config[1]
+
+        # This must be a source interface because we dont see hidservauth info
+        except IndexError:
+            source_int = onion_addr
+        # One of the files doesn't exist :| Maybe tor setup hasnt run yet.
+        except IOError:
+            return dict()
+        else:
+            journalist_int = onion_addr
+
+    return dict(journalist_location=journalist_int,
+                source_location=source_int,
+                hidserv_token=auth_token)
+
+
+if __name__ == '__main__':
+    print(json.dumps(extract_sd_onion_data()))

install_files/ansible-base/roles/app-test/tasks/extract_apptor_test_config.yml

@@ -0,0 +1,38 @@
+---
+
+- name: Create ansible fact directory
+  file:
+    path: /etc/ansible/facts.d
+    recurse: yes
+    state: directory
+    owner: root
+    group: root
+
+- name: Copy over tor application fact file
+  copy:
+    src: tor_app.fact
+    dest: /etc/ansible/facts.d/tor_app.fact
+    mode: 0755
+
+- name: Refresh remote "local facts" for glory
+  setup:
+    filter: ansible_local
+
+- name: Gather apptest facts to dict to prepare for output
+  set_fact:
+    _tbb_selenium_dict:
+      hidserv_token: "{{ ansible_local.tor_app.hidserv_token }}"
+      journalist_location: "{{ ansible_local.tor_app.journalist_location  }}"
+      source_location: "{{ ansible_local.tor_app.source_location  }}"
+      timeout: "{{ tbb_timeout }}"
+      user:
+        name: "{{ tbb_selenium_user }}"
+        password: "{{ tbb_selenium_password }}"
+        secret: "{{ tbb_selenium_totp_secret }}"
+
+- name: Dump facts to local json for in-take
+  copy:
+    content: "{{ _tbb_selenium_dict | to_nice_json }}"
+    dest: "{{ tbb_funcfolder }}"
+  become: no
+  delegate_to: localhost

install_files/ansible-base/roles/app-test/tasks/main.yml

@@ -5,3 +5,9 @@
     - apache
 
 - include: modern_gettext.yml
+  tags:
+    - modern_gettext
+
+- import_tasks: extract_apptor_test_config.yml
+  tags:
+    - functional_test

install_files/ansible-base/roles/tor-hidden-services/handlers/restart-tor-carefully.yml

@@ -48,3 +48,7 @@
     search_regex: OpenSSH
     state: started
   become: no
+
+- name: Refresh ansible local facts
+  setup:
+    filter: ansible_local

molecule/upgrade/playbook.yml

@@ -85,7 +85,6 @@
       tags: always
   become: yes
 
-
 - import_playbook: apt.yml
   tags: apt
 

securedrop/Makefile

@@ -41,10 +41,10 @@ test-config: ## Generate the test config
 	scrypt_id_pepper=$(shell head -c 32 /dev/urandom | base64) \
 	scrypt_gpg_pepper=$(shell head -c 32 /dev/urandom | base64) \
 	python -c 'import os; from jinja2 import Environment, FileSystemLoader; \
-         env = Environment(loader=FileSystemLoader(".")); \
-         ctx = {"securedrop_app_gpg_fingerprint": "65A1B5FF195B56353CC63DFFCC40EF1228271441"}; \
-         ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \
-         ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
+		 env = Environment(loader=FileSystemLoader(".")); \
+		 ctx = {"securedrop_app_gpg_fingerprint": "65A1B5FF195B56353CC63DFFCC40EF1228271441"}; \
+		 ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \
+		 ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
 	@echo >> config.py
 	@echo "SUPPORTED_LOCALES = ['ar', 'de_DE', 'es_ES', 'en_US', 'el', 'fr_FR', 'it_IT', 'nb_NO', 'nl', 'pt_BR', 'tr', 'zh_Hant']"  >> config.py
 
@@ -57,8 +57,8 @@ translate: ## Update POT translation files from sources
 	./bin/dev-shell ./i18n_tool.py translate-messages --extract-update
 	./bin/dev-shell ./i18n_tool.py translate-desktop --extract-update
 
-.PHONY: func-vnc
-func-vnc: ## Open up a VNC connection to a running docker instance
+.PHONY: docker-vnc
+docker-vnc: ## Open up a VNC connection to a running docker instance
 	../devops/scripts/vnc-docker-connect.sh
 
 # For an explanation of this shell magic, see the Makefile in the root of the repository

securedrop/bin/dev-deps

@@ -2,6 +2,10 @@
 
 export DISPLAY=:1
 
+function run_tor {
+    tor > /tmp/tor.out 2>&1 || cat /tmp/tor.out
+}
+
 function run_xvfb() {
     (setsid Xvfb :1 -screen 0 1024x768x24 -ac +extension GLX +render -noreset >& /tmp/xvfb.out || cat /tmp/xvfb.out) &
 
@@ -29,7 +33,7 @@ function setup_vncauth {
 }
 
 function run_x11vnc() {
-    setsid x11vnc -display :1 -geometry 1024x768 -rfbauth /tmp/vncpasswd -rfbport 5901 -shared >& /tmp/x11vnc.out || cat /tmp/x11vnc.out
+    setsid x11vnc -display :1 -geometry 1024x768 -rfbauth /tmp/vncpasswd -rfbport 5909 -shared >& /tmp/x11vnc.out || cat /tmp/x11vnc.out
 }
 
 function urandom() {

securedrop/bin/dev-shell

@@ -26,28 +26,32 @@ function docker_image() {
 
     docker build \
            ${DOCKER_BUILD_ARGUMENTS:-} \
-	   --build-arg=USER_ID="$(id -u)" \
-	   --build-arg=USER_NAME="${USER:-root}" \
+           --build-arg=USER_ID="$(id -u)" \
+           --build-arg=USER_NAME="${USER:-root}" \
            -t "securedrop-test-${1}-py${2}" \
-       --file "${TOPLEVEL}/securedrop/dockerfiles/${1}/python${2}/Dockerfile" \
-       "${TOPLEVEL}/securedrop"
+           --file "${TOPLEVEL}/securedrop/dockerfiles/${1}/python${2}/Dockerfile" \
+           "${TOPLEVEL}/securedrop"
 }
 
 function docker_run() {
     validate_python_version $1 $2
 
     find . \( -name '*.pyc' -o -name __pycache__ \) -delete
+
+    # The --shm-size argument sets up dedicated shared memory for the
+    # container. Our tests can fail with the default of 64m.
     docker run \
-       -p 127.0.0.1:5901:5901 \
-	   --rm \
-	   --user "${USER:-root}" \
-	   --volume "${TOPLEVEL}:${TOPLEVEL}" \
-	   --workdir "${TOPLEVEL}/securedrop" \
-       -e NUM_SOURCES \
-       -e LC_ALL=C.UTF-8 \
-       -e LANG=C.UTF-8 \
-       --name securedrop-dev \
-	   -ti ${DOCKER_RUN_ARGUMENTS:-} "securedrop-test-${1}-py${2}" "${@:3}"
+           --shm-size 2g \
+           -p 127.0.0.1:5909:5909 \
+           --rm \
+           -e NUM_SOURCES \
+           -e LC_ALL=C.UTF-8 \
+           -e LANG=C.UTF-8 \
+           --user "${USER:-root}" \
+           --volume "${TOPLEVEL}:${TOPLEVEL}" \
+           --workdir "${TOPLEVEL}/securedrop" \
+           --name securedrop-dev \
+           -ti ${DOCKER_RUN_ARGUMENTS:-} "securedrop-test-${1}-py${2}" "${@:3}"
 }
 
 docker_image $BASE_OS $PYTHON_VERSION

securedrop/bin/run

@@ -5,7 +5,6 @@ set -eu
 
 source "${BASH_SOURCE%/*}/dev-deps"
 
-run_xvfb &
 run_redis &
 run_supervisor &
 urandom