Add retry error budget (#158)

Author: chopraanmol1

httpclient/client.go

@@ -1,6 +1,7 @@
 package httpclient
 
 import (
+	"context"
 	"io"
 	"net/http"
 	"slices"
@@ -17,9 +18,10 @@ type Client struct {
 	plugins []heimdall.Plugin
 	timeout *time.Duration
 
-	retrier        heimdall.Retriable
-	retryCount     int
-	retryableCodes []int
+	retrier          heimdall.Retriable
+	retryCount       int
+	retryableCodes   []int
+	retryErrorBudget *internal.ErrorBudget
 }
 
 const (
@@ -167,7 +169,7 @@ func (c *Client) Do(request *http.Request) (*http.Response, error) {
 		if err != nil {
 			errs = append(errs, err)
 			c.reportError(request, err)
-			if internal.IsCtxDone(request.Context()) {
+			if c.skipRetry(request.Context()) {
 				break
 			}
 			continue
@@ -176,20 +178,29 @@ func (c *Client) Do(request *http.Request) (*http.Response, error) {
 
 		if _, ok := slices.BinarySearch(c.retryableCodes, response.StatusCode); ok ||
 			response.StatusCode >= http.StatusInternalServerError {
-			if internal.IsCtxDone(request.Context()) {
+			if c.skipRetry(request.Context()) {
 				break
 			}
-
 			continue
 		}
 
 		errs = nil // Clear errors if any iteration succeeds
+		_ = c.retryErrorBudget.Success()
 		break
 	}
 
 	return response, internal.BuildMultiError(errs)
 }
 
+func (c *Client) skipRetry(ctx context.Context) bool {
+	if internal.IsCtxDone(ctx) {
+		_ = c.retryErrorBudget.Success()
+		return true
+	}
+
+	return c.retryErrorBudget.Failure()
+}
+
 func (c *Client) reportRequestStart(request *http.Request) {
 	for _, plugin := range c.plugins {
 		plugin.OnRequestStart(request)

httpclient/client_test.go

@@ -307,6 +307,52 @@ func TestHTTPClientPostRetriesOnFailure(t *testing.T) {
 	assert.Equal(t, noOfCalls, count)
 }
 
+func TestHTTPClientRetriesWithBudgetGetOnFailure5xx(t *testing.T) {
+	t.Parallel()
+
+	count := 0
+	backoffInterval := 1 * time.Millisecond
+	maximumJitterInterval := 1 * time.Millisecond
+
+	client := NewClient(
+		WithHTTPTimeout(10*time.Millisecond),
+		WithRetryCount(4),
+		WithRetryErrorBudgetToken(100, 0.1),
+		WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))),
+	)
+
+	dummyHandler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{ "response": "something went wrong" }`))
+		count = count + 1
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(dummyHandler))
+	defer server.Close()
+
+	for attempt := range 10 { // under budget
+		currCount := count
+		response, err := client.Get(server.URL, http.Header{})
+		require.NoError(t, err)
+
+		assert.Equal(t, 5, count-currCount, "attempt #%d", attempt)
+		assert.NotNil(t, response)
+		assert.Equal(t, http.StatusInternalServerError, response.StatusCode)
+		assert.Equal(t, "{ \"response\": \"something went wrong\" }", respBody(t, response))
+	}
+
+	for attempt := range 10 { // over budget, no retry
+		currCount := count
+		response, err := client.Get(server.URL, http.Header{})
+		require.NoError(t, err)
+
+		assert.Equal(t, 1, count-currCount, "attempt #%d", attempt)
+		assert.NotNil(t, response)
+		assert.Equal(t, http.StatusInternalServerError, response.StatusCode)
+		assert.Equal(t, "{ \"response\": \"something went wrong\" }", respBody(t, response))
+	}
+}
+
 func BenchmarkHTTPClientPostRetriesOnFailure(b *testing.B) {
 	noOfRetries := 3
 	backoffInterval := 1 * time.Millisecond

httpclient/options.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/gojek/heimdall/v7"
+	"github.com/gojek/heimdall/v7/internal"
 )
 
 // Option represents the client options
@@ -52,3 +53,34 @@ func WithRetryableStatusCodes(statusCodes ...int) Option {
 		c.retryableCodes = codes
 	}
 }
+
+// WithRetryErrorBudgetToken creates a weighted token retry error budget with the following token details.
+//
+//	maxToken: The maximum/initial token value which is used to calculate token threshold(i.e. maxToken/2)
+//	tokenRatio: The allowed ratio of failure in comparison to success.
+func WithRetryErrorBudgetToken(maxToken int32, tokenRatio float32) Option {
+	return func(c *Client) {
+		c.retryErrorBudget = internal.NewTokenErrorBudget(maxToken, tokenRatio)
+	}
+}
+
+// WithRetryErrorBudgetPercent creates a weighted token retry error budget with the following failure details.
+//
+//	minFailureVolume: The minimum failure required.
+//	failurePercent: The failure percentage (0-100).
+//
+// Note: To determine if budget is exceeded we use recent event which satisfies following
+//
+//	failureEvent <= maxFailureEvent
+//	successEvent = (maxFailureEvent-failureEvent) / allowedSuccessPerFailure
+//	totalEvent = failureEvent + successEvent
+//
+// Where
+//
+//	maxFailureEvent = minFailureVolume * 2
+//	allowedSuccessPerFailure = (100 - failurePercent) / failurePercent
+func WithRetryErrorBudgetPercent(minFailureVolume int32, failurePercent float32) Option {
+	return func(c *Client) {
+		c.retryErrorBudget = internal.NewPercentErrorBudget(minFailureVolume, failurePercent)
+	}
+}

hystrix/hystrix_client.go

@@ -32,9 +32,10 @@ type Client struct {
 	errorPercentThreshold  int
 	fallbackFunc           func(ctx context.Context, err error) error
 
-	retrier        heimdall.Retriable
-	retryCount     int
-	retryableCodes []int
+	retrier          heimdall.Retriable
+	retryCount       int
+	retryableCodes   []int
+	retryErrorBudget *internal.ErrorBudget
 
 	statsD *plugins.StatsdCollectorConfig
 }
@@ -209,6 +210,11 @@ func (hhc *Client) Do(request *http.Request) (*http.Response, error) {
 
 		response, err = hhc.hystrixDo(request)
 		if err == nil || internal.IsCtxDone(request.Context()) {
+			_ = hhc.retryErrorBudget.Success()
+			break
+		}
+
+		if hhc.retryErrorBudget.Failure() {
 			break
 		}
 	}

hystrix/hystrix_client_test.go

@@ -334,6 +334,58 @@ func TestHystrixHTTPClientRetriesGetOnFailure5xx(t *testing.T) {
 	assert.Equal(t, "{ \"response\": \"something went wrong\" }", respBody(t, response))
 }
 
+func TestHystrixHTTPClientRetriesWithBudgetGetOnFailure5xx(t *testing.T) {
+	t.Parallel()
+
+	count := 0
+	backoffInterval := 1 * time.Millisecond
+	maximumJitterInterval := 1 * time.Millisecond
+
+	client := NewClient(
+		WithHTTPTimeout(10*time.Millisecond),
+		WithCommandName("some_command_name_5xx_with_budget"),
+		WithHystrixTimeout(10*time.Millisecond),
+		WithMaxConcurrentRequests(100),
+		WithErrorPercentThreshold(25),
+		WithSleepWindow(100),
+		WithRequestVolumeThreshold(100),
+		WithRetryCount(4),
+		WithRetryErrorBudgetToken(100, 0.1),
+		WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))),
+	)
+
+	dummyHandler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{ "response": "something went wrong" }`))
+		count = count + 1
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(dummyHandler))
+	defer server.Close()
+
+	for attempt := range 10 { // under budget
+		currCount := count
+		response, err := client.Get(server.URL, http.Header{})
+		require.NoError(t, err)
+
+		assert.Equal(t, 5, count-currCount, "attempt #%d", attempt)
+		assert.NotNil(t, response)
+		assert.Equal(t, http.StatusInternalServerError, response.StatusCode)
+		assert.Equal(t, "{ \"response\": \"something went wrong\" }", respBody(t, response))
+	}
+
+	for attempt := range 10 { // over budget, no retry
+		currCount := count
+		response, err := client.Get(server.URL, http.Header{})
+		require.NoError(t, err)
+
+		assert.Equal(t, 1, count-currCount, "attempt #%d", attempt)
+		assert.NotNil(t, response)
+		assert.Equal(t, http.StatusInternalServerError, response.StatusCode)
+		assert.Equal(t, "{ \"response\": \"something went wrong\" }", respBody(t, response))
+	}
+}
+
 func BenchmarkHystrixHTTPClientRetriesGetOnFailure(b *testing.B) {
 	backoffInterval := 1 * time.Millisecond
 	maximumJitterInterval := 1 * time.Millisecond

hystrix/options.go

@@ -8,6 +8,7 @@ import (
 	"github.com/afex/hystrix-go/plugins"
 	"github.com/gojek/heimdall/v7"
 	"github.com/gojek/heimdall/v7/httpclient"
+	"github.com/gojek/heimdall/v7/internal"
 )
 
 // Option represents the hystrix client options
@@ -118,3 +119,34 @@ func WithRetryableStatusCodes(statusCodes ...int) Option {
 		c.retryableCodes = codes
 	}
 }
+
+// WithRetryErrorBudgetToken creates a weighted token retry error budget with the following token details.
+//
+//	maxToken: The maximum/initial token value which is used to calculate token threshold(i.e. maxToken/2)
+//	tokenRatio: The allowed ratio of failure in comparison to success.
+func WithRetryErrorBudgetToken(maxToken int32, tokenRatio float32) Option {
+	return func(c *Client) {
+		c.retryErrorBudget = internal.NewTokenErrorBudget(maxToken, tokenRatio)
+	}
+}
+
+// WithRetryErrorBudgetPercent creates a weighted token retry error budget with the following failure details.
+//
+//	minFailureVolume: The minimum failure required.
+//	failurePercent: The failure percentage (0-100).
+//
+// Note: To determine if budget is exceeded we use recent event which satisfies following
+//
+//	failureEvent <= maxFailureEvent
+//	successEvent = (maxFailureEvent-failureEvent) / allowedSuccessPerFailure
+//	totalEvent = failureEvent + successEvent
+//
+// Where
+//
+//	maxFailureEvent = minFailureVolume * 2
+//	allowedSuccessPerFailure = (100 - failurePercent) / failurePercent
+func WithRetryErrorBudgetPercent(minFailureVolume int32, failurePercent float32) Option {
+	return func(c *Client) {
+		c.retryErrorBudget = internal.NewPercentErrorBudget(minFailureVolume, failurePercent)
+	}
+}