feat: Use Apache HTTPClient for downloads of public resources (#6949)

jeremylong · Sep 18, 2024 · 865e39e · 865e39e
2 parents cef8f0b + 11ae195
commit 865e39e
Show file tree

Hide file tree

Showing 36 changed files with 1,007 additions and 782 deletions.
diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Check.java
@@ -38,6 +38,8 @@
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
 import org.owasp.dependencycheck.reporting.ReportGenerator.Format;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.owasp.dependencycheck.utils.SeverityUtil;
 import org.slf4j.impl.StaticLoggerBinder;
@@ -2050,6 +2052,11 @@ protected void executeWithContextClassloader() throws BuildException {
         dealWithReferences();
         validateConfiguration();
         populateSettings();
+        try {
+            Downloader.getInstance().configure(getSettings());
+        } catch (InvalidSettingException e) {
+            throw new BuildException(e);
+        }
         try (Engine engine = new Engine(Check.class.getClassLoader(), getSettings())) {
             for (Resource resource : getPath()) {
                 final FileProvider provider = resource.as(FileProvider.class);

diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Purge.java
@@ -26,6 +26,8 @@
 import org.apache.tools.ant.Project;
 import org.apache.tools.ant.Task;
 import org.owasp.dependencycheck.Engine;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.impl.StaticLoggerBinder;
 
@@ -179,6 +181,11 @@ private void muteNoisyLoggers() {
     @SuppressWarnings("squid:RedundantThrowsDeclarationCheck")
     protected void executeWithContextClassloader() throws BuildException {
         populateSettings();
+        try {
+            Downloader.getInstance().configure(settings);
+        } catch (InvalidSettingException e) {
+            throw new BuildException(e);
+        }
         try (Engine engine = new Engine(Engine.Mode.EVIDENCE_PROCESSING, getSettings())) {
             engine.purge();
         } finally {

diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java
@@ -22,6 +22,8 @@
 import org.owasp.dependencycheck.Engine;
 import org.owasp.dependencycheck.data.nvdcve.DatabaseException;
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
+import org.owasp.dependencycheck.utils.Downloader;
+import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.impl.StaticLoggerBinder;
 
@@ -594,6 +596,11 @@ public void setHostedSuppressionsEnabled(Boolean hostedSuppressionsEnabled) {
     @Override
     protected void executeWithContextClassloader() throws BuildException {
         populateSettings();
+        try {
+            Downloader.getInstance().configure(getSettings());
+        } catch (InvalidSettingException e) {
+            throw new BuildException(e);
+        }
         try (Engine engine = new Engine(Update.class.getClassLoader(), getSettings())) {
             engine.doUpdates();
         } catch (UpdateException ex) {

diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java
@@ -33,6 +33,7 @@
 import org.owasp.dependencycheck.data.update.exception.UpdateException;
 import org.owasp.dependencycheck.exception.ExceptionCollection;
 import org.owasp.dependencycheck.exception.ReportException;
+import org.owasp.dependencycheck.utils.Downloader;
 import org.owasp.dependencycheck.utils.InvalidSettingException;
 import org.owasp.dependencycheck.utils.Settings;
 import org.slf4j.Logger;
@@ -141,6 +142,7 @@ public int run(String[] args) {
             } else {
                 try {
                     populateSettings(cli);
+                    Downloader.getInstance().configure(settings);
                 } catch (InvalidSettingException ex) {
                     LOGGER.error(ex.getMessage());
                     LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);
@@ -162,6 +164,7 @@ public int run(String[] args) {
             try {
                 populateSettings(cli);
                 settings.setBoolean(Settings.KEYS.AUTO_UPDATE, true);
+                Downloader.getInstance().configure(settings);
             } catch (InvalidSettingException ex) {
                 LOGGER.error(ex.getMessage());
                 LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);
@@ -182,6 +185,7 @@ public int run(String[] args) {
         } else if (cli.isRunScan()) {
             try {
                 populateSettings(cli);
+                Downloader.getInstance().configure(settings);
             } catch (InvalidSettingException ex) {
                 LOGGER.error(ex.getMessage(), ex);
                 LOGGER.debug(ERROR_LOADING_PROPERTIES_FILE, ex);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/AbstractSuppressionAnalyzer.java
@@ -345,14 +345,13 @@ private List<SuppressionRule> loadSuppressionFile(final SuppressionParser parser
                 deleteTempFile = true;
                 file = getSettings().getTempFile("suppression", "xml");
                 final URL url = new URL(suppressionFilePath);
-                final Downloader downloader = new Downloader(getSettings());
                 try {
-                    downloader.fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
+                    Downloader.getInstance().fetchFile(url, file, false, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
                 } catch (DownloadFailedException ex) {
                     LOGGER.trace("Failed download suppression file - first attempt", ex);
                     try {
                         Thread.sleep(500);
-                        downloader.fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
+                        Downloader.getInstance().fetchFile(url, file, true, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD);
                     } catch (TooManyRequestsException ex1) {
                         throw new SuppressionParseException("Unable to download supression file `" + file
                                 + "`; received 429 - too many requests", ex1);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/ArtifactoryAnalyzer.java
@@ -234,8 +234,7 @@ private void processPom(Dependency dependency, MavenArtifact ma) throws IOExcept
             Files.delete(pomFile.toPath());
             LOGGER.debug("Downloading {}", ma.getPomUrl());
             //TODO add caching
-            final Downloader downloader = new Downloader(getSettings());
-            downloader.fetchFile(new URL(ma.getPomUrl()), pomFile,
+            Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile, true,
                     Settings.KEYS.ANALYZER_ARTIFACTORY_API_USERNAME,
                     Settings.KEYS.ANALYZER_ARTIFACTORY_API_TOKEN);
             PomUtils.analyzePOM(dependency, pomFile);

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/CentralAnalyzer.java
@@ -241,7 +241,6 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
                                     + "this could result in undetected CPE/CVEs.", dependency.getFileName());
                             LOGGER.debug("Unable to delete temp file");
                         }
-                        final Downloader downloader = new Downloader(getSettings());
                         final int maxAttempts = this.getSettings().getInt(Settings.KEYS.ANALYZER_CENTRAL_RETRY_COUNT, 3);
                         int retryCount = 0;
                         long sleepingTimeBetweenRetriesInMillis = BASE_RETRY_WAIT;
@@ -258,7 +257,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
                             do {
                                 //CSOFF: NestedTryDepth
                                 try {
-                                    downloader.fetchFile(new URL(ma.getPomUrl()), pomFile);
+                                    Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile);
                                     success = true;
                                 } catch (DownloadFailedException ex) {
                                     try {

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/HintAnalyzer.java
@@ -267,13 +267,12 @@ private void loadHintRules() throws HintParseException {
                     deleteTempFile = true;
                     file = getSettings().getTempFile("hint", "xml");
                     final URL url = new URL(filePath);
-                    final Downloader downloader = new Downloader(getSettings());
                     try {
-                        downloader.fetchFile(url, file, false);
+                        Downloader.getInstance().fetchFile(url, file, false);
                     } catch (DownloadFailedException ex) {
                         try {
                             Thread.sleep(500);
-                            downloader.fetchFile(url, file, true);
+                            Downloader.getInstance().fetchFile(url, file, true);
                         } catch (TooManyRequestsException ex1) {
                             throw new HintParseException("Unable to download hint file `" + file + "`; received 429 - too many requests", ex1);
                         } catch (ResourceNotFoundException ex1) {

diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java
@@ -282,8 +282,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy
                         LOGGER.debug("Unable to delete temp file");
                     }
                     LOGGER.debug("Downloading {}", ma.getPomUrl());
-                    final Downloader downloader = new Downloader(getSettings());
-                    downloader.fetchFile(new URL(ma.getPomUrl()), pomFile);
+                    Downloader.getInstance().fetchFile(new URL(ma.getPomUrl()), pomFile);
                     PomUtils.analyzePOM(dependency, pomFile);
                 } catch (DownloadFailedException ex) {
                     LOGGER.warn("Unable to download pom.xml for {} from Nexus repository; "