diff options
| -rw-r--r-- | libpod/image/image.go | 7 | ||||
| -rw-r--r-- | libpod/image/pull.go | 19 | ||||
| -rw-r--r-- | vendor/github.com/containers/common/pkg/retry/retry.go | 87 | ||||
| -rw-r--r-- | vendor/modules.txt | 1 |
4 files changed, 104 insertions, 10 deletions
diff --git a/libpod/image/image.go b/libpod/image/image.go index 8b2aa318f..4664da63e 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/containers/common/pkg/retry" cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/directory" dockerarchive "github.com/containers/image/v5/docker/archive" @@ -75,6 +76,8 @@ type InfoImage struct { Layers []LayerInfo } +const maxRetry = 3 + // ImageFilter is a function to determine whether a image is included // in command output. Images to be outputted are tested using the function. // A true return will include the image, a false return will exclude it. @@ -158,7 +161,7 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, label) + imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, &retry.RetryOptions{MaxRetry: maxRetry}, label) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", name) } @@ -176,7 +179,7 @@ func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.Im if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}) + imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry}) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef)) } diff --git a/libpod/image/pull.go b/libpod/image/pull.go index d31f0dbdc..641698d03 100644 --- a/libpod/image/pull.go +++ b/libpod/image/pull.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/containers/common/pkg/retry" cp "github.com/containers/image/v5/copy" "github.com/containers/image/v5/directory" "github.com/containers/image/v5/docker" @@ -218,7 +219,7 @@ func toLocalImageName(imageName string) string { // pullImageFromHeuristicSource pulls an image based on inputName, which is heuristically parsed and may involve configured registries. // Use pullImageFromReference if the source is known precisely. -func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) { +func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) { span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromHeuristicSource") defer span.Finish() @@ -247,11 +248,11 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName) } } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, label) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, label) } // pullImageFromReference pulls an image from a types.imageReference. -func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { +func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions) ([]string, error) { span, _ := opentracing.StartSpanFromContext(ctx, "pullImageFromReference") defer span.Finish() @@ -264,7 +265,7 @@ func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.Imag if err != nil { return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef)) } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, nil) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, nil) } func cleanErrorMessage(err error) string { @@ -274,7 +275,7 @@ func cleanErrorMessage(err error) string { } // doPullImage is an internal helper interpreting pullGoal. Almost everyone should call one of the callers of doPullImage instead. -func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, label *string) ([]string, error) { +func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, retryOptions *retry.RetryOptions, label *string) ([]string, error) { span, _ := opentracing.StartSpanFromContext(ctx, "doPullImage") defer span.Finish() @@ -310,9 +311,11 @@ func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goa return nil, err } } - - _, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions) - if err != nil { + imageInfo := imageInfo + if err = retry.RetryIfNecessary(ctx, func() error { + _, err = cp.Image(ctx, policyContext, imageInfo.dstRef, imageInfo.srcRef, copyOptions) + return err + }, retryOptions); err != nil { pullErrors = multierror.Append(pullErrors, err) logrus.Debugf("Error pulling image ref %s: %v", imageInfo.srcRef.StringWithinTransport(), err) if writer != nil { diff --git a/vendor/github.com/containers/common/pkg/retry/retry.go b/vendor/github.com/containers/common/pkg/retry/retry.go new file mode 100644 index 000000000..c20f900d8 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/retry/retry.go @@ -0,0 +1,87 @@ +package retry + +import ( + "context" + "math" + "net" + "net/url" + "syscall" + "time" + + "github.com/docker/distribution/registry/api/errcode" + errcodev2 "github.com/docker/distribution/registry/api/v2" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// RetryOptions defines the option to retry +type RetryOptions struct { + MaxRetry int // The number of times to possibly retry +} + +// RetryIfNecessary retries the operation in exponential backoff with the retryOptions +func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error { + err := operation() + for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ { + delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second + logrus.Infof("Warning: failed, retrying in %s ... (%d/%d)", delay, attempt+1, retryOptions.MaxRetry) + select { + case <-time.After(delay): + break + case <-ctx.Done(): + return err + } + err = operation() + } + return err +} + +func isRetryable(err error) bool { + err = errors.Cause(err) + + if err == context.Canceled || err == context.DeadlineExceeded { + return false + } + + type unwrapper interface { + Unwrap() error + } + + switch e := err.(type) { + + case errcode.Error: + switch e.Code { + case errcode.ErrorCodeUnauthorized, errcodev2.ErrorCodeNameUnknown, errcodev2.ErrorCodeManifestUnknown: + return false + } + return true + case *net.OpError: + return isRetryable(e.Err) + case *url.Error: + return isRetryable(e.Err) + case syscall.Errno: + return e != syscall.ECONNREFUSED + case errcode.Errors: + // if this error is a group of errors, process them all in turn + for i := range e { + if !isRetryable(e[i]) { + return false + } + } + return true + case *multierror.Error: + // if this error is a group of errors, process them all in turn + for i := range e.Errors { + if !isRetryable(e.Errors[i]) { + return false + } + } + return true + case unwrapper: + err = e.Unwrap() + return isRetryable(err) + } + + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ac1e5036c..3f490616a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -90,6 +90,7 @@ github.com/containers/common/pkg/auth github.com/containers/common/pkg/capabilities github.com/containers/common/pkg/cgroupv2 github.com/containers/common/pkg/config +github.com/containers/common/pkg/retry github.com/containers/common/pkg/sysinfo github.com/containers/common/version # github.com/containers/conmon v2.0.19+incompatible |