Merge pull request #4652 from freedomofpress/onion_onion_and_more_onions

Add v3 onion support to tor-hidden-services ansible role

.circleci/config.yml

@@ -328,7 +328,7 @@ jobs:
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|docs)")
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-go
-          no_output_timeout: 20m
+          no_output_timeout: 25m
 
       - run:
           name: Ensure environment torn down

.gitignore

@@ -15,12 +15,21 @@ wheelhouse
 # ignore the instance information JSON file to prevent commit of private info
 securedrop/tests/functional/instance_information.json
 
+# ignore v3 onion JSON file
+install_files/ansible-base/tor_v3_keys.json
+
 # ignore the ATHS/THS hostname file ansible places
+# Tor v2
 app-ssh-aths
 app-document-aths # leave this here for historic reasons
 app-journalist-aths
 app-source-ths
 mon-ssh-aths
+# Tor v3
+app-journalist.auth_private
+app-sourcev3-ths
+app-ssh.auth_private
+mon-ssh.auth_private
 *.key
 *.csr
 *.pem

Makefile

@@ -37,6 +37,7 @@ update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@$(DEVSHELL) pip-compile \
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \
+		../admin/requirements.in \
 		requirements/python3/develop-requirements.in
 	@$(DEVSHELL) pip-compile \
 		--output-file requirements/python3/test-requirements.txt \
@@ -51,6 +52,7 @@ update-python2-requirements:  ## Update Python 2 requirements with pip-compile.
 	@PYTHON_VERSION=2 $(DEVSHELL) pip-compile \
 		--output-file requirements/python2/develop-requirements.txt \
 		../admin/requirements-ansible.in \
+		../admin/requirements.in \
 		requirements/python2/develop-requirements.in
 	@PYTHON_VERSION=2 $(DEVSHELL) pip-compile \
 		--output-file requirements/python2/test-requirements.txt \
@@ -60,7 +62,7 @@ update-python2-requirements:  ## Update Python 2 requirements with pip-compile.
 		requirements/python2/securedrop-app-code-requirements.in
 
 .PHONY: update-pip-requirements
-update-pip-requirements: update-admin-pip-requirements update-python3-requirements ## Update all requirements with pip-compile.
+update-pip-requirements: update-admin-pip-requirements update-python2-requirements update-python3-requirements ## Update all requirements with pip-compile.
 
 
 #################

admin/Dockerfile

@@ -17,4 +17,7 @@ ENV VIRTUAL_ENV /opt/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY requirements-dev.txt .
 RUN pip install --require-hashes -r requirements-dev.txt
+# Now also pin pip due to https://github.com/jazzband/pip-tools/issues/853
+RUN pip install pip==19.1
+
 RUN chown -R $USER_NAME /opt

admin/requirements-ansible.in

@@ -1,3 +1,3 @@
 ansible>2.6<2.7
-cryptography>=2.3
+cryptography>=2.7
 netaddr

admin/requirements.txt

@@ -70,36 +70,29 @@ cffi==1.11.4 \
     --hash=sha256:f4719d0bafc5f0a67b2ec432086d40f653840698d41fa6e9afa679403dea9d78 \
     --hash=sha256:f4992cd7b4c867f453d44c213ee29e8fd484cf81cfece4b6e836d0982b6fa1cf \
     # via bcrypt, cryptography, pynacl
-cryptography==2.3 \
-    --hash=sha256:21af753934f2f6d1a10fe8f4c0a64315af209ef6adeaee63ca349797d747d687 \
-    --hash=sha256:27bb401a20a838d6d0ea380f08c6ead3ccd8c9d8a0232dc9adcc0e4994576a66 \
-    --hash=sha256:29720c4253263cff9aea64585adbbe85013ba647f6e98367efff9db2d7193ded \
-    --hash=sha256:2a35b7570d8f247889784010aac8b384fd2e4a47b33e15c4a60b45a7c1944120 \
-    --hash=sha256:42c531a6a354407f42ee07fda5c2c0dc822cf6d52744949c182f2b295fbd4183 \
-    --hash=sha256:5eb86f03f9c4f0ac2336ac5431271072ddf7ecc76b338e26366732cfac58aa19 \
-    --hash=sha256:67f7f57eae8dede577f3f7775957f5bec93edd6bdb6ce597bb5b28e1bdf3d4fb \
-    --hash=sha256:6ec84edcbc966ae460560a51a90046503ff0b5b66157a9efc61515c68059f6c8 \
-    --hash=sha256:7ba834564daef87557e7fcd35c3c3183a4147b0b3a57314e53317360b9b201b3 \
-    --hash=sha256:7d7f084cbe1fdb82be5a0545062b59b1ad3637bc5a48612ac2eb428ff31b31ea \
-    --hash=sha256:82409f5150e529d699e5c33fa8fd85e965104db03bc564f5f4b6a9199e591f7c \
-    --hash=sha256:87d092a7c2a44e5f7414ab02fb4145723ebba411425e1a99773531dd4c0e9b8d \
-    --hash=sha256:8c56ef989342e42b9fcaba7c74b446f0cc9bed546dd00034fa7ad66fc00307ef \
-    --hash=sha256:9449f5d4d7c516a6118fa9210c4a00f34384cb1d2028672100ee0c6cce49d7f6 \
-    --hash=sha256:bc2301170986ad82d9349a91eb8884e0e191209c45f5541b16aa7c0cfb135978 \
-    --hash=sha256:c132bab45d4bd0fff1d3fe294d92b0a6eb8404e93337b3127bdec9f21de117e6 \
-    --hash=sha256:c3d945b7b577f07a477700f618f46cbc287af3a9222cd73035c6ef527ef2c363 \
-    --hash=sha256:cee18beb4c807b5c0b178f4fa2fae03cef9d51821a358c6890f8b23465b7e5d2 \
-    --hash=sha256:d01dfc5c2b3495184f683574e03c70022674ca9a7be88589c5aba130d835ea90
+cryptography==2.7 \
+    --hash=sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c \
+    --hash=sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643 \
+    --hash=sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216 \
+    --hash=sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799 \
+    --hash=sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a \
+    --hash=sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9 \
+    --hash=sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc \
+    --hash=sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8 \
+    --hash=sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53 \
+    --hash=sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1 \
+    --hash=sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609 \
+    --hash=sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292 \
+    --hash=sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e \
+    --hash=sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6 \
+    --hash=sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed \
+    --hash=sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d
 enum34==1.1.6 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
     --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
     # via cryptography
-idna==2.6 \
-    --hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f \
-    --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \
-    # via cryptography
 ipaddress==1.0.19 \
     --hash=sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81 \
     # via cryptography

admin/securedrop_admin/__init__.py

@@ -33,10 +33,14 @@
 import subprocess
 import sys
 import types
+import json
+import base64
 import prompt_toolkit
 from prompt_toolkit.validation import Validator, ValidationError
 import yaml
 from pkg_resources import parse_version
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import x25519
 
 sdlog = logging.getLogger(__name__)
 RELEASE_KEY = '22245C81E3BAEB4138B36061310F561200F4AD77'
@@ -566,6 +570,73 @@ def sdconfig(args):
     return 0
 
 
+def generate_new_v3_keys():
+    """This function generate new keys for Tor v3 onion
+    services and returns them as as tuple.
+
+    :returns: Tuple(public_key, private_key)
+    """
+
+    private_key = x25519.X25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        encoding=serialization.Encoding.Raw	,
+        format=serialization.PrivateFormat.Raw,
+        encryption_algorithm=serialization.NoEncryption())
+    public_key = private_key.public_key()
+    public_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw)
+
+    # Base32 encode and remove base32 padding characters (`=`)
+    # Using try/except blocks for Python 2/3 support.
+    try:
+        public = base64.b32encode(public_bytes).replace('=', '') \
+                       .decode("utf-8")
+    except TypeError:
+        public = base64.b32encode(public_bytes).replace(b'=', b'') \
+                       .decode("utf-8")
+    try:
+        private = base64.b32encode(private_bytes).replace('=', '') \
+                        .decode("utf-8")
+    except TypeError:
+        private = base64.b32encode(private_bytes).replace(b'=', b'') \
+                        .decode("utf-8")
+    return public, private
+
+
+def find_or_generate_new_torv3_keys(args):
+    """
+    This method will either read v3 Tor onion service keys if found or generate
+    a new public/private keypair.
+    """
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+    if os.path.exists(secret_key_path):
+        print('Tor v3 onion service keys already exist in: {}'.format(
+            secret_key_path))
+        return 0
+    # No old keys, generate and store them first
+    app_journalist_public_key, \
+        app_journalist_private_key = generate_new_v3_keys()
+    # For app ssh service
+    app_ssh_public_key, app_ssh_private_key = generate_new_v3_keys()
+    # For mon ssh service
+    mon_ssh_public_key, mon_ssh_private_key = generate_new_v3_keys()
+    tor_v3_service_info = {
+            "app_journalist_public_key": app_journalist_public_key,
+            "app_journalist_private_key": app_journalist_private_key,
+            "app_ssh_public_key": app_ssh_public_key,
+            "app_ssh_private_key": app_ssh_private_key,
+            "mon_ssh_public_key": mon_ssh_public_key,
+            "mon_ssh_private_key": mon_ssh_private_key,
+    }
+    with open(secret_key_path, 'w') as fobj:
+        json.dump(tor_v3_service_info, fobj, indent=4)
+    print('Tor v3 onion service keys generated and stored in: {}'.format(
+        secret_key_path))
+    return 0
+
+
 def install_securedrop(args):
     """Install/Update SecureDrop"""
     SiteConfig(args).load()
@@ -827,6 +898,11 @@ class ArgParseFormatterCombo(argparse.ArgumentDefaultsHelpFormatter,
                                               help=run_tails_config.__doc__)
     parse_tailsconfig.set_defaults(func=run_tails_config)
 
+    parse_generate_tor_keys = subparsers.add_parser(
+        'generate_v3_keys',
+        help=find_or_generate_new_torv3_keys.__doc__)
+    parse_generate_tor_keys.set_defaults(func=find_or_generate_new_torv3_keys)
+
     parse_backup = subparsers.add_parser('backup',
                                          help=backup_securedrop.__doc__)
     parse_backup.set_defaults(func=backup_securedrop)

admin/tests/test_securedrop-admin.py

@@ -22,6 +22,7 @@
 import argparse
 from flaky import flaky
 from os.path import dirname, join, basename, exists
+import json
 import mock
 from prompt_toolkit.validation import ValidationError
 import pytest
@@ -1008,3 +1009,58 @@ def test_load(self, caplog):
         with pytest.raises(yaml.YAMLError) as e:
             site_config.load()
         assert 'issue processing' in caplog.text
+
+
+def test_generate_new_v3_keys():
+    public, private = securedrop_admin.generate_new_v3_keys()
+
+    for key in [public, private]:
+        # base32 padding characters should be removed
+        assert '=' not in key
+        assert len(key) == 52
+
+
+def test_find_or_generate_new_torv3_keys_first_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys generated' in captured.out
+    assert return_code == 0
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    expected_keys = ['app_journalist_public_key',
+                     'app_journalist_private_key',
+                     'app_ssh_public_key',
+                     'app_ssh_private_key',
+                     'mon_ssh_public_key',
+                     'mon_ssh_private_key']
+    for key in expected_keys:
+        assert key in v3_onion_service_keys.keys()
+
+
+def test_find_or_generate_new_torv3_keys_subsequent_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+    old_keys = {'foo': 'bar'}
+    with open(secret_key_path, 'w') as f:
+        json.dump(old_keys, f)
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys already exist' in captured.out
+    assert return_code == 0
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    assert v3_onion_service_keys == old_keys

install_files/ansible-base/group_vars/all/securedrop

@@ -46,7 +46,6 @@ appserver_dependencies:
 # Enable Tor over SSH by default
 enable_ssh_over_tor: true
 
-
 # If file is present on system at the end of ansible run
 # force a reboot. Needed because of the de-coupled nature of
 # the many roles of the current prod playbook

install_files/ansible-base/group_vars/securedrop_application_server.yml

@@ -20,6 +20,17 @@ tor_instances:
   - service: journalist
     filename: app-journalist-aths
 
+tor_instances_v3:
+  - "{{ {'service': 'sshv3', 'filename': 'app-ssh.auth_private'} if enable_ssh_over_tor else [] }}"
+  - service: sourcev3
+    filename: app-sourcev3-ths
+  - service: journalistv3
+    filename: app-journalist.auth_private
+
+tor_auth_instances_v3:
+  - "{{ 'sshv3' if enable_ssh_over_tor  else [] }}"
+  - "journalistv3"
+
 authd_iprules:
   - chain: OUTPUT
     dest: "{{ monitor_ip  }}"

install_files/ansible-base/group_vars/securedrop_monitor_server.yml

@@ -14,6 +14,10 @@ local_deb_packages:
 # Configure the tor Onion Services. The Monitor server has only one,
 # for SSH, since no web interfaces.
 tor_instances: "{{ [{ 'service': 'ssh', 'filename': 'mon-ssh-aths'}] if enable_ssh_over_tor else [] }}"
+tor_instances_v3: "{{ [{ 'service': 'sshv3', 'filename': 'mon-ssh.auth_private'}] if enable_ssh_over_tor else [] }}"
+
+tor_auth_instances_v3:
+  - "{{ 'sshv3' if enable_ssh_over_tor else [] }}"
 
 authd_iprules:
   - chain: INPUT

install_files/ansible-base/group_vars/staging.yml

@@ -50,6 +50,9 @@ postfix_enable_service: no
 # Otherwise, all SSH connections would be forced over Tor.
 enable_ssh_over_tor: false
 
+# v3 onion services should be available in staging for testing.
+v3_onion_services: true
+
 ### Use for backup restores ###
 # If the `backup_zip` variable is defined ansible will copy the defined file to
 # the app server and run the 0.3_collect.py script to unzip and restore those

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.tor

@@ -51,6 +51,22 @@
   /var/lib/tor/services/hostname.tmp rw,
   /var/lib/tor/sevices/private_key rw,
   /var/lib/tor/services/private_key.tmp rw,
+  /var/lib/tor/services/sourcev3/ w,
+  /var/lib/tor/services/sourcev3/hostname rw,
+  /var/lib/tor/services/sourcev3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/sourcev3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/journalistv3/ w,
+  /var/lib/tor/services/journalistv3/hostname rw,
+  /var/lib/tor/services/journalistv3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/journalistv3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/journalistv3/authorized_keys/ w,
+  /var/lib/tor/services/journalistv3/authorized_keys/client.auth r,
+  /var/lib/tor/services/sshv3/ w,
+  /var/lib/tor/services/sshv3/hostname rw,
+  /var/lib/tor/services/sshv3/hs_ed25519_public_key rw,
+  /var/lib/tor/services/sshv3/hs_ed25519_secret_key rw,
+  /var/lib/tor/services/sshv3/authorized_keys/ w,
+  /var/lib/tor/services/sshv3/authorized_keys/client.auth r,
   /var/lib/tor/lock rwk,
   /var/lib/tor/state rw,
   /var/lib/tor/state.tmp rw,

install_files/ansible-base/roles/restrict-direct-access/defaults/main.yml

@@ -11,7 +11,6 @@ ssh_listening_address: "{{ '127.0.0.1' if enable_ssh_over_tor else '0.0.0.0' }}"
 # the hostname files can be fetched back to the Admin Workstation.
 tor_hidden_services_parent_dir: /var/lib/tor/services
 
-
 # Platform specific command for inferring the local interface utilized
 # to route to a specific IP host
 admin_net_int:
@@ -21,3 +20,23 @@ admin_net_int:
   Darwin:
     cmd: "/sbin/route -n get "
     rgx: "(?<=interface: )\\w+"
+
+# Whether to fetch back client-auth settings from the remote hosts.
+# We make this conditional to support disabling during dynamic role includes,
+# required for the ssh-over-lan strategy.
+fetch_tor_client_auth_configs: true
+
+# v2 Tor onion services are on / v3 Tor onion services are off by default for backwards
+# compatibility. Note that new install after 1.0 will have v3 enabled by sdconfig which
+# will override these variables.
+v2_onion_services: true
+v3_onion_services: false
+
+# Lookup table for querying keypair info from the local JSON
+# file on the Admin Workstation, required for configuring client
+# auth on Tor v3 Onion URLs. See the tor_v3_keys.json file for
+# reference on structure.
+tor_v3_service_map:
+  app-journalist.auth_private: "app_journalist_private_key"
+  app-ssh.auth_private: "app_ssh_private_key"
+  mon-ssh.auth_private: "mon_ssh_private_key"