Replace existing proxy config with global-agent (#572)

https://eaflood.atlassian.net/jira/software/projects/CMEA/boards/907?selectedIssue=CMEA-167

We previously fixed proxy support for requests going to the Rules Service, however this is also needed for S3 requests -- WRLS are currently having issues with sending bill runs in their local setup and this may be due to S3 requests failing for proxy reasons.

While investigating how to configure the AWS SDK to use a proxy, we discovered that the tunnel package we are using for Rules Service proxying does not appear to be currently maintained. Researching a suitable alternative led us to global-agent, which would allow us to configure the proxy in .env and have it automatically used for all outgoing requests.

We therefore implement this, and remove the existing proxy support entirely as it is no longer required. Note that the package uses the environment variable GLOBAL_AGENT_HTTP_PROXY -- the docs explain that some packages will automatically change their behaviour if HTTP_PROXY is set, so a different env var is used to prevent this from happening.

One issue we ran into was that Nock does not correctly intercept requests during unit testing after GlobalAgent.bootstrap() has been called (regardless of whether or not a proxy is defined) -- we therefore check the environment is not test before we call it.

For manual testing, we recommend enabling the nginix proxy in docker-compose.yml. Adding dns: 0.0.0.0 to the app definition is a quick way of requiring all traffic to go via the proxy and therefore verify that global-agent is working with both Rules Service and S3 requests.

* Install and set up global-agent

* Amend proxy details for manual testing

* Remove proxy in `RequestRulesServiceChargeService`

* Add globals to be ignored in testing

* Removed unused proxy config setting

* Remove unneeded tunnel package

.env.example

@@ -23,7 +23,7 @@ POSTGRES_DB_TEST=sroc_charge_test
 # Server config
 PORT=3000
 # See docker-compose.yml for directions on enabling a proxy for local testing
-HTTP_PROXY='http://proxy:3128'
+GLOBAL_AGENT_HTTP_PROXY='http://proxy:3128'
 
 # Rules service
 RULES_SERVICE_URL='https://rules.service'

.labrc.js

@@ -13,6 +13,8 @@ module.exports = {
     '__extends', '__assign', '__rest', '__decorate', '__param', '__metadata', '__awaiter', '__generator',
     '__exportStar', '__createBinding', '__values', '__read', '__spread', '__spreadArrays', '__spreadArray', '__await',
     '__asyncGenerator', '__asyncDelegator', '__asyncValues', '__makeTemplateObject', '__importStar', '__importDefault',
-    '__classPrivateFieldGet', '__classPrivateFieldSet'
+    '__classPrivateFieldGet', '__classPrivateFieldSet',
+    // We also ignore globals exposed by global-agent:
+    'GLOBAL_AGENT','ROARR'
   ].join(',')
 }

app/server.js

@@ -2,6 +2,8 @@
 
 const Hapi = require('@hapi/hapi')
 const Joi = require('joi')
+const GlobalAgent = require('global-agent')
+
 const { ServerConfig, TestConfig } = require('../config')
 const { JwtStrategyAuthLib } = require('./lib')
 const {
@@ -56,6 +58,12 @@ const registerPlugins = async (server) => {
 }
 
 const init = async () => {
+  // We use global-agent to seamlessly route requests via a proxy if one is defined in .env. However we do not want
+  // this in a test environment as it prevents Nock from successfully intercepting requests.
+  if (ServerConfig.environment !== 'test') {
+    GlobalAgent.bootstrap()
+  }
+
   // Create the hapi server
   const server = Hapi.server(ServerConfig.hapi)
 

app/services/charges/request_rules_service_charge.service.js

@@ -5,9 +5,7 @@
  */
 
 const Got = require('got')
-const Tunnel = require('tunnel')
 const Boom = require('@hapi/boom')
-const { URL } = require('url')
 
 const { RulesServiceConfig } = require('../../../config')
 
@@ -40,16 +38,15 @@ class RequestRulesServiceChargeService {
    * @returns {Object} The `response.body` provided after calling the rules service
    */
   static async go (presenter) {
-    const { url, username, password, httpProxy, timeout } = RulesServiceConfig
+    const { url, username, password, timeout } = RulesServiceConfig
     const { ruleset, regime, financialYear, chargeParams } = presenter
     const path = this._makeRulesPath(ruleset, regime, financialYear)
     const requestOptions = this._requestOptions(url, chargeParams, timeout, username, password)
-    const proxyOptions = httpProxy ? this._proxyOptions(httpProxy) : ''
 
     let response
 
     try {
-      response = await this._callRulesService(path, requestOptions, proxyOptions)
+      response = await this._callRulesService(path, requestOptions)
       await this._throwErrorIfMessagesReceived(response)
     } catch (error) {
       await this._handleErrors(error)
@@ -116,11 +113,8 @@ class RequestRulesServiceChargeService {
     return `${application}/${ruleset}${suffix}`
   }
 
-  static async _callRulesService (path, requestOptions, proxyOptions) {
-    const response = await Got.post(path, {
-      ...requestOptions,
-      ...proxyOptions
-    })
+  static async _callRulesService (path, requestOptions) {
+    const response = await Got.post(path, requestOptions)
     return response.body
   }
 
@@ -162,21 +156,6 @@ class RequestRulesServiceChargeService {
       password
     }
   }
-
-  static _proxyOptions (httpProxy) {
-    const { hostname: host, port } = new URL(httpProxy)
-
-    return {
-      agent: {
-        https: Tunnel.httpsOverHttp({
-          proxy: {
-            host,
-            port
-          }
-        })
-      }
-    }
-  }
 }
 
 module.exports = RequestRulesServiceChargeService

config/rules_service.config.js

@@ -6,7 +6,6 @@ const config = {
   url: process.env.RULES_SERVICE_URL,
   username: process.env.RULES_SERVICE_USER,
   password: process.env.RULES_SERVICE_PASSWORD,
-  httpProxy: process.env.HTTP_PROXY,
   srocMinDate: process.env.SROC_MINIMUM_DATE ? process.env.SROC_MINIMUM_DATE : '01-APR-2021',
   endpoints: {
     cfd: {

docker-compose.yml

@@ -10,7 +10,8 @@ services:
     env_file:
       - .env
 
-  # If you need to enable a proxy for testing, set HTTP_PROXY=http://proxy:3128 in .env and uncomment the following:
+  # If you need to enable a proxy for testing, set GLOBAL_AGENT_HTTP_PROXY=http://proxy:3128 in .env and uncomment the
+  # following: (See the 'app' section for details on how to require traffic to go via this proxy)
   # proxy:
   #   image: hinata/nginx-forward-proxy
   #   ports:
@@ -56,6 +57,9 @@ services:
     # up and running before starting the app. So do not use this to manage critical dependency ordering.
     depends_on:
       - db
+    # If you want to require traffic to go via a proxy, uncomment the line below:
+    # dns: 0.0.0.0
+
 
 volumes:
   cma_db_volume: