Undo some rebase mistakes, address the flakiest test

Restore reverted test-config target in securedrop/Makefile; had lost
the change from iteritems->items.

Restore molecule/ from develop.

Take a sledgehammer to journalist login steps: try logging in up to
ten times to get Selenium to see the JS after page load.

molecule/upgrade/create.yml

@@ -57,14 +57,3 @@
         content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool
-
-    - name: Ensure roles dir in-place
-      file:
-        path: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/roles"
-        state: directory
-
-    - name: Over-ride stable apt-test role logic with one from current branch
-      file:
-        src: "{{ lookup('pipe', 'git rev-parse --show-toplevel') }}/install_files/ansible-base/roles/app-test"
-        dest: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/roles/app-test"
-        state: link

molecule/vagrant-packager/destroy.yml

@@ -0,0 +1,41 @@
+---
+
+- name: Destroy
+  hosts: localhost
+  connection: local
+  vars:
+    molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}"
+    molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}"
+    molecule_scenario_dir: "{{ lookup('env','MOLECULE_SCENARIO_DIRECTORY') }}"
+    molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"
+    molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      molecule_vagrant:
+        instance_name: "{{ item.name }}"
+        platform_box: "{{ item.box }}"
+        provider_name: "{{ molecule_yml.driver.provider.name }}"
+        force_stop: "{{ item.force_stop | default(True) }}"
+
+        state: destroy
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+
+    - name: Destroy local molecule ephemeral dir
+      file:
+        state: absent
+        path: "{{ molecule_scenario_dir }}/.molecule"
+
+    # Mandatory configuration for Molecule to function.
+
+    - name: Populate instance config
+      set_fact:
+        instance_conf: {}
+
+    - name: Dump instance config
+      copy:
+        # NOTE(retr0h): Workaround for Ansible 2.2.
+        #               https://github.com/ansible/ansible/issues/20885
+        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        dest: "{{ molecule_instance_config }}"
+      when: server.changed | bool

molecule/vagrant-packager/prepare.yml

@@ -0,0 +1,9 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: False
+  tasks:
+    - name: Install python for Ansible
+      raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal)
+      become: True
+      changed_when: False

securedrop/Makefile

@@ -47,7 +47,7 @@ test-config: ## Generate the test config
 	python -c 'import os; from jinja2 import Environment, FileSystemLoader; \
 		 env = Environment(loader=FileSystemLoader(".")); \
 		 ctx = {"securedrop_app_gpg_fingerprint": "65A1B5FF195B56353CC63DFFCC40EF1228271441"}; \
-		 ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.iteritems())); \
+		 ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \
 		 ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
 	@echo >> config.py
 	@echo "SUPPORTED_LOCALES = ['ar', 'de_DE', 'es_ES', 'en_US', 'el', 'fr_FR', 'it_IT', 'nb_NO', 'nl', 'pt_BR', 'tr', 'zh_Hant']"  >> config.py

securedrop/tests/functional/journalist_navigation_steps.py

@@ -80,8 +80,7 @@ def _input_text_in_login_form(self, username, password, token):
 
     def _try_login_user(self, username, password, token):
         self._input_text_in_login_form(username, password, token)
-        submit_button = self.driver.find_element_by_css_selector('button[type="submit"]')
-        submit_button.send_keys(u"\ue007")
+        self.safe_click_by_css_selector('button[type="submit"]')
 
     def _login_user(self, username, password, otp, maxtries=3):
         token = str(otp.now())
@@ -90,7 +89,7 @@ def _login_user(self, username, password, otp, maxtries=3):
             self._try_login_user(username, password, token)
             # Successful login should redirect to the index
             self.wait_for(
-                lambda: self.driver.find_element_by_id("logout"), timeout=self.timeout + 10
+                lambda: self.driver.find_element_by_id("logout"), timeout=self.timeout * 2
             )
             if self.driver.current_url != self.journalist_location + "/":
                 new_token = str(otp.now())
@@ -117,7 +116,6 @@ def _journalist_logs_in(self):
         self.user = self.admin_user["name"]
         self.user_pw = self.admin_user["password"]
         self._login_user(self.user, self.user_pw, self.admin_user["totp"])
-
         assert self._is_on_journalist_homepage()
 
     def _journalist_visits_col(self):
@@ -730,7 +728,12 @@ def _admin_visits_edit_user(self):
         new_user_edit_links = self.driver.find_elements_by_css_selector(selector)
         assert len(new_user_edit_links) == 1
         self.safe_click_by_css_selector(selector)
-        self.wait_for(lambda: self.driver.find_element_by_id("new-password"))
+        try:
+            self.wait_for(lambda: self.driver.find_element_by_id("new-password"))
+        except NoSuchElementException:
+            # try once more
+            self.safe_click_by_css_selector(selector)
+            self.wait_for(lambda: self.driver.find_element_by_id("new-password"))
 
     def _admin_visits_reset_2fa_hotp(self):
         for i in range(3):
@@ -821,7 +824,7 @@ def _admin_enters_journalist_account_details_hotp(self, username, hotp_secret):
         hotp_checkbox.click()
 
     def _journalist_uses_js_filter_by_sources(self):
-        self.wait_for(lambda: self.driver.find_element_by_id("filter"), timeout=self.timeout * 6)
+        self.wait_for(lambda: self.driver.find_element_by_id("filter"), timeout=self.timeout * 30)
 
         filter_box = self.driver.find_element_by_id("filter")
         filter_box.send_keys("thiswordisnotinthewordlist")

securedrop/tests/functional/test_journalist.py

@@ -15,17 +15,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-from . import source_navigation_steps
-from . import journalist_navigation_steps
+import logging
+
+from selenium.common.exceptions import NoSuchElementException
+
 from . import functional_test
+from . import journalist_navigation_steps
+from . import source_navigation_steps
 
 
 class TestJournalist(
     functional_test.FunctionalTest,
     source_navigation_steps.SourceNavigationStepsMixin,
     journalist_navigation_steps.JournalistNavigationStepsMixin,
 ):
-
     def test_journalist_verifies_deletion_of_one_submission_modal(self):
         # This deletion button is displayed on the individual source page
         self._source_visits_source_homepage()
@@ -64,8 +67,21 @@ def test_journalist_interface_ui_with_modal(self):
         self._source_continues_to_submit_page()
         self._source_submits_a_file()
         self._source_logs_out()
-        self._journalist_logs_in()
-        self._journalist_uses_js_filter_by_sources()
+
+        tries = 10
+        for i in range(tries):
+            try:
+                self._journalist_logs_in()
+                self._journalist_uses_js_filter_by_sources()
+                break
+            except NoSuchElementException:
+                if i < tries:
+                    logging.error(
+                        "Journalist home page JS stymied Selenium again. Retrying page load."
+                    )
+                else:
+                    raise
+
         self._journalist_source_selection_honors_filter()
         self._journalist_selects_all_sources_then_selects_none()
         self._journalist_selects_the_first_source()