fix: introduce rate limiting, connection pooling and DNS caching [AI Augment Implemented]

build.gradle.kts

@@ -5,7 +5,7 @@ plugins {
 }
 
 group = "com.tractionrec"
-version = "1.9.0"
+version = "1.10.0"
 
 repositories {
     mavenCentral()

src/main/java/com/tractionrec/recrec/RecRecApplication.java

@@ -13,6 +13,9 @@ public class RecRecApplication {
     public static String MANIFEST_PROD_PROPERTY_NAME = "Element-Express-Production";
 
     public static void main(String[] args) {
+        // Configure DNS caching for better performance with high concurrent requests
+        configureDNSCaching();
+
         FlatLightLaf.setup();
         RecRecState state = new RecRecState();
         JFrame applicationFrame = new JFrame("RecRec");
@@ -21,7 +24,6 @@ public static void main(String[] args) {
         RecRecStart startForm = new RecRecStart(state, stack);
         stack.setInitialForm(startForm);
         stack.displayInitial();
-        System.setProperty("jdk.httpclient.connectionPoolSize", "10");
     }
 
     public static boolean isDevEnv() {
@@ -32,6 +34,23 @@ public static boolean isProduction() {
         return Manifests.exists(MANIFEST_PROD_PROPERTY_NAME) && "true".equals(Manifests.read(MANIFEST_PROD_PROPERTY_NAME));
     }
 
+    /**
+     * Configure DNS caching to prevent DNS resolution issues with high concurrent requests
+     */
+    private static void configureDNSCaching() {
+        // Cache successful DNS lookups for 5 minutes (300 seconds)
+        System.setProperty("networkaddress.cache.ttl", "300");
+
+        // Cache failed DNS lookups for 10 seconds to allow quick retry
+        System.setProperty("networkaddress.cache.negative.ttl", "10");
+
+        // Prefer IPv4 to avoid potential IPv6 resolution issues
+        System.setProperty("java.net.preferIPv4Stack", "true");
+
+        // Set connection pool size for HttpClient (moved from main method)
+        System.setProperty("jdk.httpclient.connectionPoolSize", "100");
+    }
+
     private static JMenuBar buildMenuBar() {
         JMenuBar topMenuBar = new JMenuBar();
         topMenuBar.add( Box.createHorizontalStrut( 10 ) );

src/main/java/com/tractionrec/recrec/service/AdaptiveRateLimiter.java

@@ -0,0 +1,208 @@
+package com.tractionrec.recrec.service;
+
+import com.tractionrec.recrec.RecRecApplication;
+
+import java.time.Duration;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Adaptive rate limiter that adjusts concurrency based on success/failure rates
+ * to handle API rate limiting gracefully.
+ */
+public class AdaptiveRateLimiter {
+
+    private final int minConcurrency;
+    private final int maxConcurrency;
+
+    // ✅ NEVER REPLACED - prevents deadlocks
+    private final Semaphore semaphore;
+    private final AtomicInteger targetConcurrency;
+    private final AtomicInteger successCount = new AtomicInteger(0);
+    private final AtomicInteger failureCount = new AtomicInteger(0);
+    private final AtomicLong lastAdjustmentTime = new AtomicLong(System.currentTimeMillis());
+
+    // Circuit breaker state
+    private volatile boolean circuitOpen = false;
+    private volatile long circuitOpenTime = 0;
+    private final AtomicInteger consecutiveFailures = new AtomicInteger(0);
+    private static final Duration CIRCUIT_RECOVERY_TIME = Duration.ofSeconds(30);
+    private static final int FAILURE_THRESHOLD = 10; // Open circuit after 10 consecutive failures
+    private static final Duration ADJUSTMENT_INTERVAL = Duration.ofSeconds(30); // Adjust every 30 seconds (more conservative)
+
+    public AdaptiveRateLimiter(int initialConcurrency, int minConcurrency, int maxConcurrency) {
+        this.minConcurrency = minConcurrency;
+        this.maxConcurrency = maxConcurrency;
+        this.targetConcurrency = new AtomicInteger(initialConcurrency);
+        this.semaphore = new Semaphore(initialConcurrency, true); // ✅ Created once, never replaced
+    }
+
+    /**
+     * Acquire a permit to make a request
+     */
+    public void acquire() throws InterruptedException {
+        // Check circuit breaker
+        if (circuitOpen) {
+            if (System.currentTimeMillis() - circuitOpenTime > CIRCUIT_RECOVERY_TIME.toMillis()) {
+                // Try to close circuit and allow this request to proceed
+                closeCircuit();
+            } else {
+                // Circuit is open, wait before allowing request
+                Thread.sleep(1000);
+                throw new RuntimeException("Circuit breaker is open - API appears to be overwhelmed");
+            }
+        }
+
+        semaphore.acquire();
+    }
+
+    /**
+     * Release a permit after request completion
+     */
+    public void release() {
+        semaphore.release();
+    }
+
+    /**
+     * Record a successful request
+     */
+    public void recordSuccess() {
+        successCount.incrementAndGet();
+        consecutiveFailures.set(0); // Reset consecutive failure count on success
+        adjustConcurrencyIfNeeded();
+    }
+
+    /**
+     * Record a failed request (timeout, connection error, etc.)
+     */
+    public void recordFailure() {
+        failureCount.incrementAndGet();
+        int consecutive = consecutiveFailures.incrementAndGet();
+
+        // Open circuit breaker if too many consecutive failures
+        if (consecutive >= FAILURE_THRESHOLD && !circuitOpen) {
+            openCircuit();
+        }
+
+        adjustConcurrencyIfNeeded();
+    }
+
+    /**
+     * Adjust concurrency based on success/failure rates
+     */
+    private void adjustConcurrencyIfNeeded() {
+        long now = System.currentTimeMillis();
+        long lastAdjustment = lastAdjustmentTime.get();
+
+        if (now - lastAdjustment < ADJUSTMENT_INTERVAL.toMillis()) {
+            return; // Too soon to adjust
+        }
+
+        if (!lastAdjustmentTime.compareAndSet(lastAdjustment, now)) {
+            return; // Another thread is adjusting
+        }
+
+        int successes = successCount.getAndSet(0);
+        int failures = failureCount.getAndSet(0);
+        int total = successes + failures;
+
+        if (total < 5) {
+            return; // Not enough data to make adjustment
+        }
+
+        double failureRate = (double) failures / total;
+        int current = targetConcurrency.get();
+        int newConcurrency = current;
+
+        if (failureRate > 0.3) { // More than 30% failure rate (more conservative)
+            // Reduce concurrency in increments of 5
+            newConcurrency = Math.max(minConcurrency, current - 5);
+            System.err.printf("High failure rate (%.1f%%), reducing concurrency from %d to %d%n",
+                failureRate * 100, current, newConcurrency);
+        } else if (failureRate < 0.05 && current < maxConcurrency) { // Less than 5% failure rate
+            // Gradually increase concurrency in increments of 5
+            newConcurrency = Math.min(maxConcurrency, current + 5);
+            System.out.printf("Low failure rate (%.1f%%), increasing concurrency from %d to %d%n",
+                failureRate * 100, current, newConcurrency);
+        }
+
+        if (newConcurrency != current) {
+            updateConcurrency(newConcurrency);
+        }
+    }
+
+    /**
+     * Update the concurrency limit safely without replacing semaphore
+     * ✅ DEADLOCK-FREE: Adjusts permits on existing semaphore
+     */
+    private void updateConcurrency(int newConcurrency) {
+        int oldConcurrency = targetConcurrency.getAndSet(newConcurrency);
+        int delta = newConcurrency - oldConcurrency;
+
+        if (delta > 0) {
+            // Increase concurrency: add permits
+            semaphore.release(delta);
+            System.out.printf("Increased concurrency from %d to %d (+%d permits)%n",
+                oldConcurrency, newConcurrency, delta);
+        } else if (delta < 0) {
+            // Decrease concurrency: drain and reset permits
+            int drained = semaphore.drainPermits();
+            semaphore.release(Math.max(0, newConcurrency));
+            System.out.printf("Decreased concurrency from %d to %d (drained %d, released %d)%n",
+                oldConcurrency, newConcurrency, drained, Math.max(0, newConcurrency));
+        }
+    }
+
+    /**
+     * Open the circuit breaker
+     */
+    private void openCircuit() {
+        circuitOpen = true;
+        circuitOpenTime = System.currentTimeMillis();
+        System.err.println("Circuit breaker OPENED - too many failures, pausing requests");
+
+        // Drastically reduce concurrency when circuit opens
+        updateConcurrency(Math.max(1, minConcurrency / 2));
+    }
+
+    /**
+     * Close the circuit breaker
+     */
+    private void closeCircuit() {
+        circuitOpen = false;
+        System.out.println("Circuit breaker CLOSED - resuming normal operation");
+
+        // Reset to minimum concurrency when circuit closes
+        updateConcurrency(minConcurrency);
+        successCount.set(0);
+        failureCount.set(0);
+        consecutiveFailures.set(0);
+    }
+
+    /**
+     * Get current concurrency limit
+     */
+    public int getCurrentConcurrency() {
+        return targetConcurrency.get();
+    }
+
+    /**
+     * Check if circuit breaker is open
+     */
+    public boolean isCircuitOpen() {
+        return circuitOpen;
+    }
+
+    /**
+     * Get current success/failure statistics
+     */
+    public String getStats() {
+        return String.format("Concurrency: %d, Circuit: %s, Success: %d, Failures: %d, Consecutive: %d",
+            getCurrentConcurrency(),
+            circuitOpen ? "OPEN" : "CLOSED",
+            successCount.get(),
+            failureCount.get(),
+            consecutiveFailures.get());
+    }
+}

src/main/java/com/tractionrec/recrec/service/BINQueryService.java

@@ -10,9 +10,9 @@
 import gg.jte.output.StringOutput;
 
 import java.net.URI;
-import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -24,16 +24,16 @@ private BINQueryService(Boolean isProduction, TemplateEngine templateEngine) {
     }
 
     public BINQueryResult queryForBINInfo(String accountId, String accountToken, QueryItem item) {
-        HttpClient client = HttpClient.newHttpClient();
         String requestBody = getBINBody(accountId, accountToken, item);
         HttpRequest request = HttpRequest.newBuilder()
                 .uri(getTransactionURI())
                 .POST(HttpRequest.BodyPublishers.ofString(requestBody))
                 .header("SOAPAction", "https://transaction.elementexpress.com/EnhancedBINQuery")
                 .header("Content-Type", "text/xml")
+                .timeout(Duration.ofSeconds(60)) // Request timeout
                 .build();
         try {
-            HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+            HttpResponse<String> response = executeRequestWithRetry(request);
             if(response.statusCode() != 200) {
                 return new BINQueryResult(item, ResultStatus.ERROR, "Status Code: " + response.statusCode());
             }

src/main/java/com/tractionrec/recrec/service/PaymentAccountQueryService.java

@@ -12,9 +12,9 @@
 import org.apache.commons.text.StringEscapeUtils;
 
 import java.net.URI;
-import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 
@@ -25,39 +25,36 @@ private PaymentAccountQueryService(Boolean isProduction, TemplateEngine template
     }
 
     public PaymentAccountQueryResult queryForPaymentAccount(String accountId, String accountToken, QueryItem item) {
-        try (HttpClient client = HttpClient.newHttpClient()) {
-            String requestBody = getPaBody(accountId, accountToken, item);
-            HttpRequest request = HttpRequest.newBuilder()
-                    .uri(getServicesUri())
-                    .POST(HttpRequest.BodyPublishers.ofString(requestBody))
-                    .header("SOAPAction", "https://services.elementexpress.com/PaymentAccountQuery")
-                    .header("Content-Type", "text/xml")
-                    .build();
-            try {
-                HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-                if (response.statusCode() != 200) {
-                    return new PaymentAccountQueryResult(item, ResultStatus.ERROR, "Status Code: " + response.statusCode());
-                }
-                final String responseBody = response.body();
-                final String responseElement = responseBody.substring(responseBody.indexOf("<response>"), responseBody.indexOf("</PaymentAccountQueryResponse>")).trim();
+        String requestBody = getPaBody(accountId, accountToken, item);
+        HttpRequest request = HttpRequest.newBuilder()
+                .uri(getServicesUri())
+                .POST(HttpRequest.BodyPublishers.ofString(requestBody))
+                .header("SOAPAction", "https://services.elementexpress.com/PaymentAccountQuery")
+                .header("Content-Type", "text/xml")
+                .timeout(Duration.ofSeconds(60)) // Request timeout
+                .build();
+        try {
+            HttpResponse<String> response = executeRequestWithRetry(request);
+            if (response.statusCode() != 200) {
+                return new PaymentAccountQueryResult(item, ResultStatus.ERROR, "Status Code: " + response.statusCode());
+            }
+            final String responseBody = response.body();
+            final String responseElement = responseBody.substring(responseBody.indexOf("<response>"), responseBody.indexOf("</PaymentAccountQueryResponse>")).trim();
 
-                final PaymentAccountQueryResponse queryResponse = mapper.readValue(responseElement, PaymentAccountQueryResponse.class);
-                if (queryResponse.responseCode == 90) {
-                    return new PaymentAccountQueryResult(item, ResultStatus.NOT_FOUND, queryResponse.responseMessage);
-                }
-                if (queryResponse.responseCode != 0) {
-                    return new PaymentAccountQueryResult(item, ResultStatus.ERROR, queryResponse.responseMessage);
-                }
-                final String unescapedQueryData = StringEscapeUtils.unescapeXml(queryResponse.queryData);
-                final List<PaymentAccount> results = mapper.readValue(unescapedQueryData, new TypeReference<>() {
-                });
-                return new PaymentAccountQueryResult(item, ResultStatus.SUCCESS, queryResponse.responseMessage, results);
-            } catch (Exception ex) {
-                ex.printStackTrace();
-                return new PaymentAccountQueryResult(item, ResultStatus.ERROR, ex.getMessage());
-            } finally {
-                client.close();
+            final PaymentAccountQueryResponse queryResponse = mapper.readValue(responseElement, PaymentAccountQueryResponse.class);
+            if (queryResponse.responseCode == 90) {
+                return new PaymentAccountQueryResult(item, ResultStatus.NOT_FOUND, queryResponse.responseMessage);
+            }
+            if (queryResponse.responseCode != 0) {
+                return new PaymentAccountQueryResult(item, ResultStatus.ERROR, queryResponse.responseMessage);
             }
+            final String unescapedQueryData = StringEscapeUtils.unescapeXml(queryResponse.queryData);
+            final List<PaymentAccount> results = mapper.readValue(unescapedQueryData, new TypeReference<>() {
+            });
+            return new PaymentAccountQueryResult(item, ResultStatus.SUCCESS, queryResponse.responseMessage, results);
+        } catch (Exception ex) {
+            ex.printStackTrace();
+            return new PaymentAccountQueryResult(item, ResultStatus.ERROR, ex.getMessage());
         }
     }