Retry pulling image

Wrap the inner helper in the retry function. Functions pullimage failed with retriable error will default maxretry 3 times using exponential backoff.

Signed-off-by: Qi Wang <qiwan@redhat.com>

4 files changed, 104 insertions, 10 deletions

diff --git a/libpod/image/image.go b/libpod/image/image.go
index 8b2aa318f..4664da63e 100644
--- a/libpod/image/image.go
+++ b/libpod/image/image.go
@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containers/common/pkg/retry"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
@@ -75,6 +76,8 @@ type InfoImage struct {
 	Layers []LayerInfo
 }
 
+const maxRetry = 3
+
 // ImageFilter is a function to determine whether a image is included
 // in command output. Images to be outputted are tested using the function.
 // A true return will include the image, a false return will exclude it.
@@ -158,7 +161,7 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, label)
+	imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, &retry.RetryOptions{MaxRetry: maxRetry}, label)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to pull %s", name)
 	}
@@ -176,7 +179,7 @@ func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.Im
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{})
+	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
 	}
diff --git a/libpod/image/pull.go b/libpod/image/pull.go
index d31f0dbdc..641698d03 100644
--- a/libpod/image/pull.go
+++ b/libpod/image/pull.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/containers/common/pkg/retry"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
 	"github.com/containers/image/v5/docker"
@@ -218,7 +219,7 @@ func toLocalImageName(imageName string) string {
 
 // pullImageFromHeuristicSource pulls an image based on inputName, which is heuristically parsed and may involve configured registries.
 // Use pullImageFromReference if the source is known precisely.
-func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) {
+func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromHeuristicSource")
 	defer span.Finish()
 
@@ -247,11 +248,11 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s
 			return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName)
 		}
 	}
-	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, label)
+	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, label)
 }
 
 // pullImageFromReference pulls an image from a types.imageReference.
-func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) {
+func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions) ([]string, error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromReference")
 	defer span.Finish()
 
@@ -264,7 +265,7 @@ func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.Imag
 	if err != nil {
 		return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef))
 	}
-	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, nil)
+	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, nil)
 }
 
 func cleanErrorMessage(err error) string {
@@ -274,7 +275,7 @@ func cleanErrorMessage(err error) string {
 }
 
 // doPullImage is an internal helper interpreting pullGoal. Almost everyone should call one of the callers of doPullImage instead.
-func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) {
+func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "doPullImage")
 	defer span.Finish()
 
@@ -310,9 +311,11 @@ func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goa
 				return nil, err
 			}
 		}
-
-		_, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions)
-		if err != nil {
+		imageInfo := imageInfo
+		if err = retry.RetryIfNecessary(ctx, func() error {
+			_, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions)
+			return err
+		}, retryOptions); err != nil {
 			pullErrors = multierror.Append(pullErrors, err)
 			logrus.Debugf("Error pulling image ref %s: %v", imageInfo.srcRef.StringWithinTransport(), err)
 			if writer != nil {
diff --git a/vendor/github.com/containers/common/pkg/retry/retry.go b/vendor/github.com/containers/common/pkg/retry/retry.go
new file mode 100644
index 000000000..c20f900d8
--- /dev/null
+++ b/vendor/github.com/containers/common/pkg/retry/retry.go
@@ -0,0 +1,87 @@
+package retry
+
+import (
+	"context"
+	"math"
+	"net"
+	"net/url"
+	"syscall"
+	"time"
+
+	"github.com/docker/distribution/registry/api/errcode"
+	errcodev2 "github.com/docker/distribution/registry/api/v2"
+	"github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// RetryOptions defines the option to retry
+type RetryOptions struct {
+	MaxRetry int // The number of times to possibly retry
+}
+
+// RetryIfNecessary retries the operation in exponential backoff with the retryOptions
+func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error {
+	err := operation()
+	for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ {
+		delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second
+		logrus.Infof("Warning: failed, retrying in %s ... (%d/%d)", delay, attempt+1, retryOptions.MaxRetry)
+		select {
+		case <-time.After(delay):
+			break
+		case <-ctx.Done():
+			return err
+		}
+		err = operation()
+	}
+	return err
+}
+
+func isRetryable(err error) bool {
+	err = errors.Cause(err)
+
+	if err == context.Canceled || err == context.DeadlineExceeded {
+		return false
+	}
+
+	type unwrapper interface {
+		Unwrap() error
+	}
+
+	switch e := err.(type) {
+
+	case errcode.Error:
+		switch e.Code {
+		case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown:
+			return false
+		}
+		return true
+	case *net.OpError:
+		return isRetryable(e.Err)
+	case *url.Error:
+		return isRetryable(e.Err)
+	case syscall.Errno:
+		return e != syscall.ECONNREFUSED
+	case errcode.Errors:
+		// if this error is a group of errors, process them all in turn
+		for i := range e {
+			if !isRetryable(e[i]) {
+				return false
+			}
+		}
+		return true
+	case *multierror.Error:
+		// if this error is a group of errors, process them all in turn
+		for i := range e.Errors {
+			if !isRetryable(e.Errors[i]) {
+				return false
+			}
+		}
+		return true
+	case unwrapper:
+		err = e.Unwrap()
+		return isRetryable(err)
+	}
+
+	return false
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index ac1e5036c..3f490616a 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -90,6 +90,7 @@ github.com/containers/common/pkg/auth
 github.com/containers/common/pkg/capabilities
 github.com/containers/common/pkg/cgroupv2
 github.com/containers/common/pkg/config
+github.com/containers/common/pkg/retry
 github.com/containers/common/pkg/sysinfo
 github.com/containers/common/version
 # github.com/containers/conmon v2.0.19+incompatible