Deploy RC 390 to Production

.gitlab-ci.yml

@@ -214,6 +214,29 @@ check_changelog:
         exit 0
       fi
 
+migrate:
+  stage: test
+  needs:
+    - job: install
+  cache:
+    - <<: *ruby_cache
+  variables:
+    DOCKER_DB_HOST: db-postgres
+    POSTGRES_DB: identity_idp_test
+    POSTGRES_USER: postgres_user
+    POSTGRES_PASSWORD: postgres_password
+    POSTGRES_HOST_AUTH_METHOD: trust
+    RAILS_ENV: test
+  services:
+    - name: postgres:13.9
+      alias: db-postgres
+      command: ['--fsync=false', '--synchronous_commit=false', '--full_page_writes=false']
+  script:
+    - *bundle_install
+    - bundle exec rake db:create db:migrate --trace
+    - git diff db/
+    - make lint_database_schema_files
+
 specs:
   stage: test
   needs:
@@ -319,6 +342,8 @@ js_tests:
     - yarn test
 
 pinpoint-check:
+  needs:
+    - job: install
   stage: test
   cache:
     - <<: *ruby_cache
@@ -328,6 +353,18 @@ pinpoint-check:
     - *yarn_install
     - make lint_country_dialing_codes
 
+audit_packages:
+  needs:
+    - job: install
+  stage: test
+  cache:
+    - <<: *ruby_cache
+    - <<: *yarn_cache
+  script:
+    - *bundle_install
+    - *yarn_install
+    - make audit
+
 prepare_deploy:
   # Runs in parallel with tests so we can deploy more quickly after passing
   stage: test
@@ -354,6 +391,7 @@ coverage:
     - *bundle_install
     - bundle exec spec/simplecov_merger.rb
     - mv coverage/coverage/* coverage/
+  coverage: '/LOC \(\d+.\d+\%\) covered/'
   artifacts:
     reports:
       coverage_report:
@@ -413,7 +451,7 @@ trigger_devops:
     name: dtzar/helm-kubectl:latest
   script:
     - kubectl config get-contexts
-    - export CONTEXT=$(kubectl config get-contexts | grep review-apps | awk '{print $1}' | head -1)
+    - export CONTEXT=$(kubectl config get-contexts | grep reviewapp | awk '{print $1}' | head -1)
     - kubectl config use-context "$CONTEXT"
     - |-
       export IDP_CONFIG=$(cat <<EOF
@@ -434,16 +472,16 @@ trigger_devops:
           "throttleUrl": "redis://$CI_ENVIRONMENT_SLUG-login-chart-redis.review-apps:6379/1",
           "url": "redis://$CI_ENVIRONMENT_SLUG-login-chart-redis.review-apps:6379"
         },
-        "assetHost": "https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
-        "domainName": "$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
+        "assetHost": "https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
+        "domainName": "$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
         "loginDatacenter": "true",
         "loginDomain": "identitysandbox.gov",
         "loginEnv": "$CI_ENVIRONMENT_SLUG",
         "loginHostRole": "idp",
         "loginSkipRemoteConfig": "true",
-        "pivcacServiceUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov/",
-        "pivcacVerifyTokenUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov/",
-        "dashboardUrl": "https://$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov"
+        "pivcacServiceUrl": "https://$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov/",
+        "pivcacVerifyTokenUrl": "https://$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov/",
+        "dashboardUrl": "https://$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov"
       }
       EOF
       )
@@ -466,30 +504,30 @@ trigger_devops:
           "throttleUrl": "redis://$CI_ENVIRONMENT_SLUG-login-chart-redis.review-apps:6379/1",
           "url": "redis://$CI_ENVIRONMENT_SLUG-login-chart-redis.review-apps:6379"
         },
-        "assetHost": "https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
-        "domainName": "$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
+        "assetHost": "https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
+        "domainName": "$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
         "loginDatacenter": "true",
         "loginDomain": "identitysandbox.gov",
         "loginEnv": "$CI_ENVIRONMENT_SLUG",
         "loginHostRole": "worker",
         "loginSkipRemoteConfig": "true",
-        "pivcacServiceUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov/",
-        "pivcacVerifyTokenUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov/"
+        "pivcacServiceUrl": "https://$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov/",
+        "pivcacVerifyTokenUrl": "https://$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov/"
       }
       EOF
       )
     - |-
       export PIVCAC_CONFIG=$(cat <<EOF
       {
         "kubernetesReviewApp": "true",
-        "clientCertS3Bucket": "login-gov-pivcac-public-cert-reviewapps.894947205914-us-west-2",
+        "clientCertS3Bucket": "login-gov-pivcac-public-cert-reviewapp.894947205914-us-west-2",
         "postgres": {
           "sslmode": "prefer",
           "name": "idp",
           "host": "$CI_ENVIRONMENT_SLUG-login-chart-pivcac-pg.review-apps"
         },
-        "idpHost": "$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
-        "domainName": "$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov"
+        "idpHost": "$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
+        "domainName": "$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov"
       }
       EOF
       )
@@ -505,19 +543,19 @@ trigger_devops:
         "newrelic": {
           "enabled": "false"
         },
-        "samlSpIssuer": "https://$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov",
-        "idpUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
-        "idpSpUrl": "https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov",
-        "postLogoutUrl": "https://$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov",
-        "domainName": "$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov"
+        "samlSpIssuer": "https://$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov",
+        "idpUrl": "https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
+        "idpSpUrl": "https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov",
+        "postLogoutUrl": "https://$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov",
+        "domainName": "$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov"
       }
       EOF
       )
     - git clone -b main --single-branch https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.login.gov/lg-public/identity-idp-helm-chart.git
     - >-
       helm upgrade --install --namespace review-apps
       --debug
-      --set env="reviewapps-$CI_ENVIRONMENT_SLUG"
+      --set env="reviewapp-$CI_ENVIRONMENT_SLUG"
       --set idp.image.repository="${ECR_REGISTRY}/identity-idp/review"
       --set idp.image.tag="${CI_COMMIT_SHA}"
       --set worker.image.repository="${ECR_REGISTRY}/identity-idp/review"
@@ -533,31 +571,31 @@ trigger_devops:
       --set-json idp.config="$IDP_CONFIG"
       --set-json worker.config="$WORKER_CONFIG"
       --set-json pivcac.config="$PIVCAC_CONFIG"
-      --set-json idp.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
-      --set-json pivcac.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
-      --set-json dashboard.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
+      --set-json idp.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
+      --set-json pivcac.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
+      --set-json dashboard.ingress.hosts="[{\"host\": \"$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov\", \"paths\": [{\"path\": \"/\", \"pathType\": \"Prefix\"}]}]"
       $CI_ENVIRONMENT_SLUG ./identity-idp-helm-chart
     - echo "DNS may take a while to propagate, so be patient if it doesn't show up right away"
-    - echo "To access the rails console, first run 'aws-vault exec sandbox-power -- aws eks update-kubeconfig --name reviewapps'"
+    - echo "To access the rails console, first run 'aws-vault exec sandbox-power -- aws eks update-kubeconfig --name reviewapp'"
     - echo "Then run aws-vault exec sandbox-power -- kubectl exec -it service/$CI_ENVIRONMENT_SLUG-login-chart-idp -n review-apps -- /app/bin/rails console"
     - echo "Address of IDP review app:"
-    - echo https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+    - echo https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
     - echo "Address of PIVCAC review app:"
-    - echo https://$CI_ENVIRONMENT_SLUG.review-app.pivcac.identitysandbox.gov
+    - echo https://$CI_ENVIRONMENT_SLUG.pivcac.reviewapp.identitysandbox.gov
     - echo "Address of Dashboard review app:"
-    - echo https://$CI_ENVIRONMENT_SLUG-review-app-dashboard.review-app.identitysandbox.gov
+    - echo https://$CI_ENVIRONMENT_SLUG-dashboard.reviewapp.identitysandbox.gov
 
 
 review-app:
   stage: review
   allow_failure: true
   needs:
     - job: build-review-image
-  resource_group: $CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+  resource_group: $CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
   extends: .deploy
   environment:
     name: review/$CI_COMMIT_REF_NAME
-    url: https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+    url: https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
     on_stop: stop-review-app
     auto_stop_in: 2 days
   rules:
@@ -566,9 +604,9 @@ review-app:
       when: never
 
 stop-review-app:
-  resource_group: $CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+  resource_group: $CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
   script:
-    - export CONTEXT=$(kubectl config get-contexts | grep review-apps | awk '{print $1}' | head -1)
+    - export CONTEXT=$(kubectl config get-contexts | grep reviewapp | awk '{print $1}' | head -1)
     - kubectl config use-context "$CONTEXT"
     - helm uninstall --namespace review-apps $CI_ENVIRONMENT_SLUG
   stage: review
@@ -590,12 +628,12 @@ deploy_production:
   allow_failure: true
   needs:
     - job: build-review-image
-  resource_group: $CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+  resource_group: $CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
   extends: .deploy
   environment:
     name: production
     deployment_tier: production
-    url: https://$CI_ENVIRONMENT_SLUG.review-app.identitysandbox.gov
+    url: https://$CI_ENVIRONMENT_SLUG.reviewapp.identitysandbox.gov
   rules:
     - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
 

Gemfile

@@ -71,7 +71,7 @@ gem 'rqrcode'
 gem 'ruby-progressbar'
 gem 'ruby-saml'
 gem 'safe_target_blank', '>= 1.0.2'
-gem 'saml_idp', github: '18F/saml_idp', tag: '0.21.2-18f'
+gem 'saml_idp', github: '18F/saml_idp', tag: '0.21.4-18f'
 gem 'scrypt'
 gem 'simple_form', '>= 5.0.2'
 gem 'stringex', require: false

Gemfile.lock

@@ -35,10 +35,10 @@ GIT
 
 GIT
   remote: https://github.com/18F/saml_idp.git
-  revision: 5ad9e188efdfa6597697dd87f9cb9e8efa8d7d09
-  tag: 0.21.2-18f
+  revision: 5e9999ef8e9260cda74cfea0a637f754994e0f9d
+  tag: 0.21.4-18f
   specs:
-    saml_idp (0.21.2.pre.18f)
+    saml_idp (0.21.4.pre.18f)
       activesupport
       builder
       faraday

Makefile

@@ -12,6 +12,7 @@ ARTIFACT_DESTINATION_FILE ?= ./tmp/idp.tar.gz
 
 .PHONY: \
 	analytics_events \
+	audit \
 	brakeman \
 	build_artifact \
 	check \
@@ -25,6 +26,7 @@ ARTIFACT_DESTINATION_FILE ?= ./tmp/idp.tar.gz
 	lint_analytics_events \
 	lint_analytics_events_sorted \
 	lint_country_dialing_codes \
+	lint_database_schema_files \
 	lint_erb \
 	lint_font_glyphs \
 	lint_lockfiles \
@@ -74,11 +76,7 @@ endif
 	make lint_analytics_events_sorted
 	@echo "--- brakeman ---"
 	make brakeman
-	@echo "--- bundler-audit ---"
-	bundle exec bundler-audit check --update
 	# JavaScript
-	@echo "--- yarn audit ---"
-	yarn audit --groups dependencies; test $$? -le 7
 	@echo "--- eslint ---"
 	yarn run lint
 	@echo "--- typescript ---"
@@ -105,6 +103,12 @@ endif
 	@echo "--- lint migrations ---"
 	make lint_migrations
 
+audit: ## Checks packages for vulnerabilities
+	@echo "--- bundler-audit ---"
+	bundle exec bundler-audit check --update
+	@echo "--- yarn audit ---"
+	yarn audit --groups dependencies; test $$? -le 7
+
 lint_erb: ## Lints ERB files
 	bundle exec erblint app/views app/components
 
@@ -253,6 +257,10 @@ update_pinpoint_supported_countries: ## Updates list of countries supported by P
 lint_country_dialing_codes: update_pinpoint_supported_countries ## Checks that countries supported by Pinpoint for voice and SMS are up to date
 	(! git diff --name-only | grep config/country_dialing_codes.yml) || (echo "Error: Run 'make update_pinpoint_supported_countries' to update country codes"; exit 1)
 
+lint_database_schema_files: ## Checks that database schema files have not changed
+	(! git diff --name-only | grep db/schema.rb) || (echo "Error: db/schema.rb does not match after running migrations"; exit 1)
+	(! git diff --name-only | grep db/worker_jobs_schema.rb) || (echo "Error: db/worker_jobs_schema.rb does not match after running migrations"; exit 1)
+
 build_artifact $(ARTIFACT_DESTINATION_FILE): ## Builds zipped tar file artifact with IDP source code and Ruby/JS dependencies
 	@echo "Building artifact into $(ARTIFACT_DESTINATION_FILE)"
 	bundle config set --local cache_all true

app/controllers/concerns/idv/verify_info_concern.rb

@@ -24,13 +24,7 @@ def shared_update
 
       idv_session.verify_info_step_document_capture_session_uuid = document_capture_session.uuid
 
-      # proof_resolution job expects these values
-      agent_pii = pii.merge(
-        uuid: current_user.uuid,
-        uuid_prefix: ServiceProvider.find_by(issuer: sp_session[:issuer])&.app_id,
-        ssn: idv_session.ssn,
-      )
-      Idv::Agent.new(agent_pii).proof_resolution(
+      Idv::Agent.new(pii).proof_resolution(
         document_capture_session,
         should_proof_state_id: aamva_state?,
         trace_id: amzn_trace_id,
@@ -164,8 +158,6 @@ def process_async_state(current_async_state)
     end
 
     def async_state_done(current_async_state)
-      add_proofing_costs(current_async_state.result)
-
       create_fraud_review_request_if_needed(current_async_state.result)
 
       form_response = idv_result_to_form_response(
@@ -295,34 +287,6 @@ def move_applicant_to_idv_session
       idv_session.applicant['uuid'] = current_user.uuid
     end
 
-    def add_proofing_costs(results)
-      return if results[:context][:sp_costs_added]
-
-      results[:context][:stages].each do |stage, hash|
-        if stage == :resolution
-          # transaction_id comes from ConversationId
-          add_cost(:lexis_nexis_resolution, transaction_id: hash[:transaction_id])
-        elsif stage == :residential_address
-          next if pii[:same_address_as_id] == 'true'
-          next if hash[:vendor_name] == 'ResidentialAddressNotRequired'
-          add_cost(:lexis_nexis_resolution, transaction_id: hash[:transaction_id])
-        elsif stage == :state_id
-          next if hash[:exception].present?
-          next if hash[:vendor_name] == 'UnsupportedJurisdiction'
-          # transaction_id comes from TransactionLocatorId
-          add_cost(:aamva, transaction_id: hash[:transaction_id])
-        elsif stage == :threatmetrix
-          # transaction_id comes from request_id
-          if hash[:transaction_id]
-            add_cost(
-              :threatmetrix,
-              transaction_id: hash[:transaction_id],
-            )
-          end
-        end
-      end
-    end
-
     def add_cost(token, transaction_id: nil)
       Db::SpCost::AddSpCost.call(current_sp, token, transaction_id: transaction_id)
     end

app/controllers/idv/address_controller.rb

@@ -20,7 +20,6 @@ def update
       @address_form = build_address_form
       form_result = @address_form.submit(profile_params)
       track_submit_event(form_result)
-      capture_address_edited(form_result)
       if form_result.success?
         success
       else
@@ -69,8 +68,11 @@ def failure
     end
 
     def track_submit_event(form_result)
-      address_edited = form_result.success? && address_edited?
-      analytics.idv_address_submitted(**form_result.to_h.merge(address_edited:))
+      analytics.idv_address_submitted(
+        **form_result.to_h.merge(
+          address_edited: address_edited?,
+        ),
+      )
     end
 
     def address_edited?
@@ -80,10 +82,5 @@ def address_edited?
     def profile_params
       params.require(:idv_form).permit(Idv::AddressForm::ATTRIBUTES)
     end
-
-    def capture_address_edited(result)
-      address_edited = result.to_h[:address_edited]
-      idv_session.address_edited = true if address_edited
-    end
   end
 end