Implement rate limiting retryer

When receiving a rate limit response from AWS, the application would
exit with an error. This was an undesirable outcome.

AWS has a feature to implement retrying using their `retry`
[package][1].

By default there are a number of retryable conditions. The relevant
useful cases are as follows:

- Connection Errors
- RequestTimeout, RequestTimeoutException
- Throttling, ThrottlingException, ThrottledException,
  RequestThrottledException, TooManyRequestsException,
  RequestThrottled, SlowDown
- RequestLimitExceeded, BandwidthLimitExceeded, LimitExceededException

This means the default retryer will handle rate limiting and will not
need to implicitly handle this case.

It is also important to note the following about client rate limiting:

> Generally you will always want to return new instance of a Retryer.
> This will avoid a global rate limit bucket being shared across all
> service clients.

This means that the instantiation of the S3 client must be handled
within each Go routine and cannot be shared as it was previously
implemented.

`AdaptiveMode` is the retry strategy that will be used:

> AdaptiveMode provides an experimental retry strategy that expands on
> the Standard retry strategy, adding client attempt rate limits. The
> attempt rate limit is initially unrestricted, but becomes restricted
> when the attempt fails with for a throttle error.

The default, values for the `AdaptiveMode` is based on the
`NewStandard` which is:

- `DefaultMaxAttempts`: `3`
- `DefaultMaxBackoff`: `20s`

[1]: https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/aws/retry

Fixes: #21
Signed-off-by: Michael Lorant <[email protected]>

main.go

@@ -125,8 +125,7 @@ func main() {
 	fmt.Println("🌎 bucket located in", bucketRegion)
 	fmt.Println("")
 
-	// recreate s3svc with bucket's region, create cloudwatch svc
-	s3svc = s3.NewService(s3.WithAWSEndpoint(cli.AWSEndpoint), s3.WithRegion(bucketRegion))
+	// create cloudwatch svc
 	cloudwatchSvc := cloudwatch.NewService(cloudwatch.WithAWSEndpoint(cli.AWSEndpoint), cloudwatch.WithRegion(bucketRegion))
 
 	// Fetch bucket metrics
@@ -172,7 +171,7 @@ func main() {
 	println("")
 
 	log.Debug().Str("bucket", selectedBucket).Int("concurrency", cli.Concurrency).Msg("starting nuke")
-	err = nuke(ctx, s3svc, selectedBucket, cli.Concurrency)
+	err = nuke(ctx, cli.AWSEndpoint, selectedBucket, bucketRegion, cli.Concurrency)
 	if err != nil {
 		fmt.Println("error:", err)
 		os.Exit(1)
@@ -181,7 +180,7 @@ func main() {
 }
 
 // Delete operation w/progress bar
-func nuke(ctx context.Context, s3svc s3.Service, bucket string, concurrency int) error {
+func nuke(ctx context.Context, awsEndpoint, bucket, bucketRegion string, concurrency int) error {
 	fmt.Println("")
 
 	c := counter.New()
@@ -228,6 +227,9 @@ func nuke(ctx context.Context, s3svc s3.Service, bucket string, concurrency int)
 	g.Go(func() error {
 		defer close(s3DeleteQueue)
 
+		// Create new S3 service for queueing objects.
+		s3svc := s3.NewService(s3.WithAWSEndpoint(awsEndpoint), s3.WithRegion(bucketRegion))
+
 		c, err := workers.S3QueueObjectVersions(ctx, s3svc, bucket, s3DeleteQueue)
 		if err != nil {
 			return err
@@ -239,6 +241,10 @@ func nuke(ctx context.Context, s3svc s3.Service, bucket string, concurrency int)
 
 	for i := 0; i < concurrency; i++ {
 		g.Go(func() error {
+			// Create new S3 service for each worker. This is necessary to avoid a global rate limit bucket
+			// being shared between all service clients.
+			s3svc := s3.NewService(s3.WithAWSEndpoint(awsEndpoint), s3.WithRegion(bucketRegion))
+
 			deleteCount, err := workers.S3DeleteFromChannel(ctx, s3svc, bucket, s3DeleteQueue, deleteProgress, deleteFailures)
 			if err != nil {
 				return err

pkg/aws/config/config.go

@@ -4,10 +4,15 @@ import (
 	"context"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/aws/aws-sdk-go-v2/config"
 )
 
 // New creates a new aws.Config with custom endpoint resolver and region set
 func New(region string) (aws.Config, error) {
-	return config.LoadDefaultConfig(context.TODO(), config.WithRegion(region))
+	retrier := func() aws.Retryer {
+		return retry.NewAdaptiveMode()
+	}
+
+	return config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), config.WithRetryer(retrier))
 }