fix: add retry on 503 errors (jeremylong#90)

Author: jeremylong

buildSrc/src/main/groovy/vuln.tools.java-common-conventions.gradle

@@ -12,7 +12,7 @@ plugins {
 }
 
 group 'io.github.jeremylong'
-version = '5.0.1'
+version = '5.0.2'
 
 repositories {
     mavenCentral()

open-vulnerability-clients/README.md

@@ -39,14 +39,14 @@ See API usage examples in the [open-vulnerability-store](https://github.com/jere
 <dependency>
    <groupId>io.github.jeremylong</groupId>
    <artifactId>open-vulnerability-clients</artifactId>
-   <version>5.0.1</version>
+   <version>5.0.2</version>
 </dependency>
 ```
 
 ### gradle
 
 ```groovy
-implementation 'io.github.jeremylong:open-vulnerability-clients:5.0.1'
+implementation 'io.github.jeremylong:open-vulnerability-clients:5.0.2'
 ```
 
 ### api usage

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdApiRetryStrategy.java

@@ -0,0 +1,59 @@
+/*
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2023 Jeremy Long. All Rights Reserved.
+ */
+package io.github.jeremylong.openvulnerability.client.nvd;
+
+import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.protocol.HttpContext;
+import org.apache.hc.core5.util.TimeValue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Implements a back-off delay retry strategy.
+ */
+public class NvdApiRetryStrategy extends DefaultHttpRequestRetryStrategy {
+    /**
+     * Maximum number of allowed retries.
+     */
+    private final int maxRetries;
+
+    /**
+     * Retry interval between subsequent retries in milliseconds.
+     */
+    private final long delay;
+
+    public NvdApiRetryStrategy(int maxRetries, long delay) {
+        super(maxRetries, TimeValue.of(delay, TimeUnit.MILLISECONDS), new ArrayList<Class<? extends IOException>>(),
+                Arrays.asList(503));
+        this.maxRetries = maxRetries;
+        this.delay = delay;
+    }
+
+    @Override
+    public TimeValue getRetryInterval(final HttpResponse response, final int execCount, final HttpContext context) {
+
+        if (execCount < maxRetries / 2) {
+            return TimeValue.of(delay * execCount, TimeUnit.MILLISECONDS);
+        }
+
+        return TimeValue.of(delay * execCount / 2, TimeUnit.MILLISECONDS);
+    }
+}

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java

@@ -133,7 +133,20 @@ public class NvdCveClient implements PagedDataSource<DefCveItem> {
      * @param maxPageCount the maximum number of pages to retrieve from the NVD API.
      */
     NvdCveClient(String apiKey, String endpoint, int threadCount, int maxPageCount) {
-        this(apiKey, endpoint, apiKey == null ? 6500 : 600, threadCount, maxPageCount);
+        this(apiKey, endpoint, apiKey == null ? 6500 : 600, threadCount, maxPageCount, 10);
+    }
+
+    /**
+     * Constructs a new NVD CVE API client.
+     *
+     * @param apiKey the api key; can be null
+     * @param endpoint the endpoint for the NVD CVE API; if null the default endpoint is used
+     * @param threadCount the number of threads to use when calling the NVD API.
+     * @param maxPageCount the maximum number of pages to retrieve from the NVD API.
+     * @param maxRetryCount the maximum number of retries for 503 status code responses.
+     */
+    NvdCveClient(String apiKey, String endpoint, int threadCount, int maxPageCount, int maxRetryCount) {
+        this(apiKey, endpoint, apiKey == null ? 6500 : 600, threadCount, maxPageCount, maxRetryCount);
     }
 
     /**
@@ -144,8 +157,9 @@ public class NvdCveClient implements PagedDataSource<DefCveItem> {
      * @param delay the delay in milliseconds between API calls on a single thread.
      * @param threadCount the number of threads to use when calling the NVD API.
      * @param maxPageCount the maximum number of pages to retrieve from the NVD API.
+     * @param maxRetryCount the maximum number of retries for 503 status code responses.
      */
-    NvdCveClient(String apiKey, String endpoint, long delay, int threadCount, int maxPageCount) {
+    NvdCveClient(String apiKey, String endpoint, long delay, int threadCount, int maxPageCount, int maxRetryCount) {
         this.apiKey = apiKey;
         if (endpoint == null) {
             this.endpoint = DEFAULT_ENDPOINT;
@@ -173,7 +187,7 @@ public class NvdCveClient implements PagedDataSource<DefCveItem> {
         }
         clients = new ArrayList<>(threadCount);
         for (int i = 0; i < threadCount; i++) {
-            clients.add(new RateLimitedClient(delay, meter));
+            clients.add(new RateLimitedClient(maxRetryCount, delay, meter));
         }
         objectMapper = new ObjectMapper();
         objectMapper.registerModule(new JavaTimeModule());

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClientBuilder.java

@@ -64,6 +64,10 @@ public final class NvdCveClientBuilder {
      * The minimum delay between API calls in milliseconds.
      */
     private long delay;
+    /**
+     * The maximum number of retries for 503 responses from the NVD.
+     */
+    private int maxRetryCount = 10;
     /**
      * The number of threads to use when calling the NVD API.
      */
@@ -122,6 +126,17 @@ public NvdCveClientBuilder withDelay(long milliseconds) {
         return this;
     }
 
+    /**
+     * Set the maximum number of retries for 503 responses from the NVD; default is 10.
+     *
+     * @param maxRetryCount the maximum number of retries for 503 responses from the NVD.
+     * @return the builder
+     */
+    public NvdCveClientBuilder withMaxRetryCount(int maxRetryCount) {
+        this.maxRetryCount = maxRetryCount;
+        return this;
+    }
+
     /**
      * Set the number of threads to use when calling the NVD API.
      *
@@ -322,9 +337,9 @@ public NvdCveClientBuilder withVersionEnd(String versionEnd, VersionType endType
     public NvdCveClient build() {
         NvdCveClient client;
         if (delay > 0) {
-            client = new NvdCveClient(apiKey, endpoint, delay, threadCount, maxPageCount);
+            client = new NvdCveClient(apiKey, endpoint, delay, threadCount, maxPageCount, maxRetryCount);
         } else {
-            client = new NvdCveClient(apiKey, endpoint, threadCount, maxPageCount);
+            client = new NvdCveClient(apiKey, endpoint, threadCount, maxPageCount, maxRetryCount);
         }
         if (!filters.isEmpty()) {
             client.setFilters(filters);

open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/RateLimitedClient.java

@@ -107,12 +107,25 @@ class RateLimitedClient implements AutoCloseable {
      * @param meter the rate meter to limit the request rate
      */
     RateLimitedClient(long minimumDelay, RateMeter meter) {
+        this(10, minimumDelay, meter);
+    }
+
+    /**
+     * Construct a rate limited client with a given delay and request window configuration. This allows callers to
+     * configure 5 requests are allowed over a 30-second rolling window and we will delay at least 4 seconds between
+     * calls to help more evenly distribute the calls across the request window.
+     *
+     * @param maxRetries the maximum number of retry attemps
+     * @param minimumDelay the number of milliseconds to wait between API calls
+     * @param meter the rate meter to limit the request rate
+     */
+    RateLimitedClient(int maxRetries, long minimumDelay, RateMeter meter) {
         this.meter = meter;
         this.delay = minimumDelay;
         LOG.debug("rate limited call delay: {}", delay);
-
+        NvdApiRetryStrategy retryStrategy = new NvdApiRetryStrategy(maxRetries, minimumDelay);
         SystemDefaultRoutePlanner planner = new SystemDefaultRoutePlanner(ProxySelector.getDefault());
-        client = HttpAsyncClients.custom().setRoutePlanner(planner).build();
+        client = HttpAsyncClients.custom().setRoutePlanner(planner).setRetryStrategy(retryStrategy).build();
         client.start();
     }
 

vulnz/README.md

@@ -72,7 +72,7 @@ export JAVA_OPTS="-Xmx2g"
 Alternatively, run the CLI using the `-Xmx2g` argument:
 
 ```bash
-java -Xmx2g -jar ./vulnz-5.0.1.jar
+java -Xmx2g -jar ./vulnz-5.0.2.jar
 ```
 
 ### Creating the Cache
@@ -89,7 +89,7 @@ for file in *.json; do gzip -k "${file}"; done
 Alternatively, without using the above install command:
 
 ```bash
-./vulnz-5.0.1.jar cve --cache --directory ./cache
+./vulnz-5.0.2.jar cve --cache --directory ./cache
 cd cache
 for file in *.json; do gzip -k "${file}"; done 
 ```

vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/AbstractNvdCommand.java

@@ -28,6 +28,9 @@ public abstract class AbstractNvdCommand extends AbstractJsonCommand {
     @CommandLine.Option(names = {
             "--delay"}, description = "The delay in milliseconds between API calls to the NVD - important if pulling a larger data set without an API Key")
     private int delay;
+    @CommandLine.Option(names = {
+            "--maxRetry"}, description = "The maximum number of retry attempts on 503 errors from the NVD API")
+    private int maxRetry;
     @CommandLine.Option(names = {
             "--pageCount"}, description = "The number of `pages` of data to retrieve from the NVD if more then a single page is returned")
     private int pageCount = 0;
@@ -59,6 +62,10 @@ protected int getDelay() {
         return delay;
     }
 
+    protected int getMaxRetry() {
+        return maxRetry;
+    }
+
     /**
      * Returns the NVD API Key if supplied.
      *

vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java

@@ -113,6 +113,9 @@ public Integer timedCall() throws Exception {
         if (getDelay() > 0) {
             builder.withDelay(getDelay());
         }
+        if (getMaxRetry() > 0) {
+            builder.withMaxRetryCount(getMaxRetry());
+        }
         if (cveId != null) {
             builder.withFilter(NvdCveClientBuilder.Filter.CVE_ID, cveId);
         }