diff options
Diffstat (limited to 'vendor')
66 files changed, 3848 insertions, 417 deletions
diff --git a/vendor/github.com/containerd/containerd/log/context.go b/vendor/github.com/containerd/containerd/log/context.go new file mode 100644 index 000000000..31f1a3ac0 --- /dev/null +++ b/vendor/github.com/containerd/containerd/log/context.go @@ -0,0 +1,90 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package log + +import ( + "context" + "sync/atomic" + + "github.com/sirupsen/logrus" +) + +var ( + // G is an alias for GetLogger. + // + // We may want to define this locally to a package to get package tagged log + // messages. + G = GetLogger + + // L is an alias for the standard logger. + L = logrus.NewEntry(logrus.StandardLogger()) +) + +type ( + loggerKey struct{} +) + +// TraceLevel is the log level for tracing. Trace level is lower than debug level, +// and is usually used to trace detailed behavior of the program. +const TraceLevel = logrus.Level(uint32(logrus.DebugLevel + 1)) + +// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to +// ensure the formatted time is always the same number of characters. +const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00" + +// ParseLevel takes a string level and returns the Logrus log level constant. +// It supports trace level. +func ParseLevel(lvl string) (logrus.Level, error) { + if lvl == "trace" { + return TraceLevel, nil + } + return logrus.ParseLevel(lvl) +} + +// WithLogger returns a new context with the provided logger. Use in +// combination with logger.WithField(s) for great effect. +func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context { + return context.WithValue(ctx, loggerKey{}, logger) +} + +// GetLogger retrieves the current logger from the context. If no logger is +// available, the default logger is returned. +func GetLogger(ctx context.Context) *logrus.Entry { + logger := ctx.Value(loggerKey{}) + + if logger == nil { + return L + } + + return logger.(*logrus.Entry) +} + +// Trace logs a message at level Trace with the log entry passed-in. +func Trace(e *logrus.Entry, args ...interface{}) { + level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level))) + if level >= TraceLevel { + e.Debug(args...) + } +} + +// Tracef logs a message at level Trace with the log entry passed-in. +func Tracef(e *logrus.Entry, format string, args ...interface{}) { + level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level))) + if level >= TraceLevel { + e.Debugf(format, args...) + } +} diff --git a/vendor/github.com/containerd/containerd/platforms/compare.go b/vendor/github.com/containerd/containerd/platforms/compare.go new file mode 100644 index 000000000..3ad22a10d --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/compare.go @@ -0,0 +1,229 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import specs "github.com/opencontainers/image-spec/specs-go/v1" + +// MatchComparer is able to match and compare platforms to +// filter and sort platforms. +type MatchComparer interface { + Matcher + + Less(specs.Platform, specs.Platform) bool +} + +// Only returns a match comparer for a single platform +// using default resolution logic for the platform. +// +// For ARMv8, will also match ARMv7, ARMv6 and ARMv5 (for 32bit runtimes) +// For ARMv7, will also match ARMv6 and ARMv5 +// For ARMv6, will also match ARMv5 +func Only(platform specs.Platform) MatchComparer { + platform = Normalize(platform) + if platform.Architecture == "arm" { + if platform.Variant == "v8" { + return orderedPlatformComparer{ + matchers: []Matcher{ + &matcher{ + Platform: platform, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v7", + }, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v6", + }, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v5", + }, + }, + }, + } + } + if platform.Variant == "v7" { + return orderedPlatformComparer{ + matchers: []Matcher{ + &matcher{ + Platform: platform, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v6", + }, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v5", + }, + }, + }, + } + } + if platform.Variant == "v6" { + return orderedPlatformComparer{ + matchers: []Matcher{ + &matcher{ + Platform: platform, + }, + &matcher{ + Platform: specs.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: "v5", + }, + }, + }, + } + } + } + + return singlePlatformComparer{ + Matcher: &matcher{ + Platform: platform, + }, + } +} + +// Ordered returns a platform MatchComparer which matches any of the platforms +// but orders them in order they are provided. +func Ordered(platforms ...specs.Platform) MatchComparer { + matchers := make([]Matcher, len(platforms)) + for i := range platforms { + matchers[i] = NewMatcher(platforms[i]) + } + return orderedPlatformComparer{ + matchers: matchers, + } +} + +// Any returns a platform MatchComparer which matches any of the platforms +// with no preference for ordering. +func Any(platforms ...specs.Platform) MatchComparer { + matchers := make([]Matcher, len(platforms)) + for i := range platforms { + matchers[i] = NewMatcher(platforms[i]) + } + return anyPlatformComparer{ + matchers: matchers, + } +} + +// All is a platform MatchComparer which matches all platforms +// with preference for ordering. +var All MatchComparer = allPlatformComparer{} + +type singlePlatformComparer struct { + Matcher +} + +func (c singlePlatformComparer) Less(p1, p2 specs.Platform) bool { + return c.Match(p1) && !c.Match(p2) +} + +type orderedPlatformComparer struct { + matchers []Matcher +} + +func (c orderedPlatformComparer) Match(platform specs.Platform) bool { + for _, m := range c.matchers { + if m.Match(platform) { + return true + } + } + return false +} + +func (c orderedPlatformComparer) Less(p1 specs.Platform, p2 specs.Platform) bool { + for _, m := range c.matchers { + p1m := m.Match(p1) + p2m := m.Match(p2) + if p1m && !p2m { + return true + } + if p1m || p2m { + return false + } + } + return false +} + +type anyPlatformComparer struct { + matchers []Matcher +} + +func (c anyPlatformComparer) Match(platform specs.Platform) bool { + for _, m := range c.matchers { + if m.Match(platform) { + return true + } + } + return false +} + +func (c anyPlatformComparer) Less(p1, p2 specs.Platform) bool { + var p1m, p2m bool + for _, m := range c.matchers { + if !p1m && m.Match(p1) { + p1m = true + } + if !p2m && m.Match(p2) { + p2m = true + } + if p1m && p2m { + return false + } + } + // If one matches, and the other does, sort match first + return p1m && !p2m +} + +type allPlatformComparer struct{} + +func (allPlatformComparer) Match(specs.Platform) bool { + return true +} + +func (allPlatformComparer) Less(specs.Platform, specs.Platform) bool { + return false +} diff --git a/vendor/github.com/containerd/containerd/platforms/cpuinfo.go b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go new file mode 100644 index 000000000..69b336d67 --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/cpuinfo.go @@ -0,0 +1,117 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "bufio" + "os" + "runtime" + "strings" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/log" + "github.com/pkg/errors" +) + +// Present the ARM instruction set architecture, eg: v7, v8 +var cpuVariant string + +func init() { + if isArmArch(runtime.GOARCH) { + cpuVariant = getCPUVariant() + } else { + cpuVariant = "" + } +} + +// For Linux, the kernel has already detected the ABI, ISA and Features. +// So we don't need to access the ARM registers to detect platform information +// by ourselves. We can just parse these information from /proc/cpuinfo +func getCPUInfo(pattern string) (info string, err error) { + if !isLinuxOS(runtime.GOOS) { + return "", errors.Wrapf(errdefs.ErrNotImplemented, "getCPUInfo for OS %s", runtime.GOOS) + } + + cpuinfo, err := os.Open("/proc/cpuinfo") + if err != nil { + return "", err + } + defer cpuinfo.Close() + + // Start to Parse the Cpuinfo line by line. For SMP SoC, we parse + // the first core is enough. + scanner := bufio.NewScanner(cpuinfo) + for scanner.Scan() { + newline := scanner.Text() + list := strings.Split(newline, ":") + + if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) { + return strings.TrimSpace(list[1]), nil + } + } + + // Check whether the scanner encountered errors + err = scanner.Err() + if err != nil { + return "", err + } + + return "", errors.Wrapf(errdefs.ErrNotFound, "getCPUInfo for pattern: %s", pattern) +} + +func getCPUVariant() string { + if runtime.GOOS == "windows" { + // Windows only supports v7 for ARM32 and v8 for ARM64 and so we can use + // runtime.GOARCH to determine the variants + var variant string + switch runtime.GOARCH { + case "arm64": + variant = "v8" + case "arm": + variant = "v7" + default: + variant = "unknown" + } + + return variant + } + + variant, err := getCPUInfo("Cpu architecture") + if err != nil { + log.L.WithError(err).Error("failure getting variant") + return "" + } + + switch variant { + case "8", "AArch64": + variant = "v8" + case "7", "7M", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)": + variant = "v7" + case "6", "6TEJ": + variant = "v6" + case "5", "5T", "5TE", "5TEJ": + variant = "v5" + case "4", "4T": + variant = "v4" + case "3": + variant = "v3" + default: + variant = "unknown" + } + + return variant +} diff --git a/vendor/github.com/containerd/containerd/platforms/database.go b/vendor/github.com/containerd/containerd/platforms/database.go new file mode 100644 index 000000000..6ede94061 --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/database.go @@ -0,0 +1,114 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "runtime" + "strings" +) + +// isLinuxOS returns true if the operating system is Linux. +// +// The OS value should be normalized before calling this function. +func isLinuxOS(os string) bool { + return os == "linux" +} + +// These function are generated from https://golang.org/src/go/build/syslist.go. +// +// We use switch statements because they are slightly faster than map lookups +// and use a little less memory. + +// isKnownOS returns true if we know about the operating system. +// +// The OS value should be normalized before calling this function. +func isKnownOS(os string) bool { + switch os { + case "aix", "android", "darwin", "dragonfly", "freebsd", "hurd", "illumos", "js", "linux", "nacl", "netbsd", "openbsd", "plan9", "solaris", "windows", "zos": + return true + } + return false +} + +// isArmArch returns true if the architecture is ARM. +// +// The arch value should be normalized before being passed to this function. +func isArmArch(arch string) bool { + switch arch { + case "arm", "arm64": + return true + } + return false +} + +// isKnownArch returns true if we know about the architecture. +// +// The arch value should be normalized before being passed to this function. +func isKnownArch(arch string) bool { + switch arch { + case "386", "amd64", "amd64p32", "arm", "armbe", "arm64", "arm64be", "ppc64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "mips64p32", "mips64p32le", "ppc", "riscv", "riscv64", "s390", "s390x", "sparc", "sparc64", "wasm": + return true + } + return false +} + +func normalizeOS(os string) string { + if os == "" { + return runtime.GOOS + } + os = strings.ToLower(os) + + switch os { + case "macos": + os = "darwin" + } + return os +} + +// normalizeArch normalizes the architecture. +func normalizeArch(arch, variant string) (string, string) { + arch, variant = strings.ToLower(arch), strings.ToLower(variant) + switch arch { + case "i386": + arch = "386" + variant = "" + case "x86_64", "x86-64": + arch = "amd64" + variant = "" + case "aarch64", "arm64": + arch = "arm64" + switch variant { + case "8", "v8": + variant = "" + } + case "armhf": + arch = "arm" + variant = "v7" + case "armel": + arch = "arm" + variant = "v6" + case "arm": + switch variant { + case "", "7": + variant = "v7" + case "5", "6", "8": + variant = "v" + variant + } + } + + return arch, variant +} diff --git a/vendor/github.com/containerd/containerd/platforms/defaults.go b/vendor/github.com/containerd/containerd/platforms/defaults.go new file mode 100644 index 000000000..a14d80e58 --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/defaults.go @@ -0,0 +1,38 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + "runtime" + + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +// DefaultString returns the default string specifier for the platform. +func DefaultString() string { + return Format(DefaultSpec()) +} + +// DefaultSpec returns the current platform's default platform specification. +func DefaultSpec() specs.Platform { + return specs.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + // The Variant field will be empty if arch != ARM. + Variant: cpuVariant, + } +} diff --git a/vendor/github.com/containerd/containerd/platforms/defaults_unix.go b/vendor/github.com/containerd/containerd/platforms/defaults_unix.go new file mode 100644 index 000000000..e8a7d5ffa --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/defaults_unix.go @@ -0,0 +1,24 @@ +// +build !windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +// Default returns the default matcher for the platform. +func Default() MatchComparer { + return Only(DefaultSpec()) +} diff --git a/vendor/github.com/containerd/containerd/platforms/defaults_windows.go b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go new file mode 100644 index 000000000..0defbd36c --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/defaults_windows.go @@ -0,0 +1,31 @@ +// +build windows + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package platforms + +import ( + specs "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Default returns the default matcher for the platform. +func Default() MatchComparer { + return Ordered(DefaultSpec(), specs.Platform{ + OS: "linux", + Architecture: "amd64", + }) +} diff --git a/vendor/github.com/containerd/containerd/platforms/platforms.go b/vendor/github.com/containerd/containerd/platforms/platforms.go new file mode 100644 index 000000000..d2b73ac3d --- /dev/null +++ b/vendor/github.com/containerd/containerd/platforms/platforms.go @@ -0,0 +1,279 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package platforms provides a toolkit for normalizing, matching and +// specifying container platforms. +// +// Centered around OCI platform specifications, we define a string-based +// specifier syntax that can be used for user input. With a specifier, users +// only need to specify the parts of the platform that are relevant to their +// context, providing an operating system or architecture or both. +// +// How do I use this package? +// +// The vast majority of use cases should simply use the match function with +// user input. The first step is to parse a specifier into a matcher: +// +// m, err := Parse("linux") +// if err != nil { ... } +// +// Once you have a matcher, use it to match against the platform declared by a +// component, typically from an image or runtime. Since extracting an images +// platform is a little more involved, we'll use an example against the +// platform default: +// +// if ok := m.Match(Default()); !ok { /* doesn't match */ } +// +// This can be composed in loops for resolving runtimes or used as a filter for +// fetch and select images. +// +// More details of the specifier syntax and platform spec follow. +// +// Declaring Platform Support +// +// Components that have strict platform requirements should use the OCI +// platform specification to declare their support. Typically, this will be +// images and runtimes that should make these declaring which platform they +// support specifically. This looks roughly as follows: +// +// type Platform struct { +// Architecture string +// OS string +// Variant string +// } +// +// Most images and runtimes should at least set Architecture and OS, according +// to their GOARCH and GOOS values, respectively (follow the OCI image +// specification when in doubt). ARM should set variant under certain +// discussions, which are outlined below. +// +// Platform Specifiers +// +// While the OCI platform specifications provide a tool for components to +// specify structured information, user input typically doesn't need the full +// context and much can be inferred. To solve this problem, we introduced +// "specifiers". A specifier has the format +// `<os>|<arch>|<os>/<arch>[/<variant>]`. The user can provide either the +// operating system or the architecture or both. +// +// An example of a common specifier is `linux/amd64`. If the host has a default +// of runtime that matches this, the user can simply provide the component that +// matters. For example, if a image provides amd64 and arm64 support, the +// operating system, `linux` can be inferred, so they only have to provide +// `arm64` or `amd64`. Similar behavior is implemented for operating systems, +// where the architecture may be known but a runtime may support images from +// different operating systems. +// +// Normalization +// +// Because not all users are familiar with the way the Go runtime represents +// platforms, several normalizations have been provided to make this package +// easier to user. +// +// The following are performed for architectures: +// +// Value Normalized +// aarch64 arm64 +// armhf arm +// armel arm/v6 +// i386 386 +// x86_64 amd64 +// x86-64 amd64 +// +// We also normalize the operating system `macos` to `darwin`. +// +// ARM Support +// +// To qualify ARM architecture, the Variant field is used to qualify the arm +// version. The most common arm version, v7, is represented without the variant +// unless it is explicitly provided. This is treated as equivalent to armhf. A +// previous architecture, armel, will be normalized to arm/v6. +// +// While these normalizations are provided, their support on arm platforms has +// not yet been fully implemented and tested. +package platforms + +import ( + "regexp" + "runtime" + "strconv" + "strings" + + "github.com/containerd/containerd/errdefs" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +var ( + specifierRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +) + +// Matcher matches platforms specifications, provided by an image or runtime. +type Matcher interface { + Match(platform specs.Platform) bool +} + +// NewMatcher returns a simple matcher based on the provided platform +// specification. The returned matcher only looks for equality based on os, +// architecture and variant. +// +// One may implement their own matcher if this doesn't provide the required +// functionality. +// +// Applications should opt to use `Match` over directly parsing specifiers. +func NewMatcher(platform specs.Platform) Matcher { + return &matcher{ + Platform: Normalize(platform), + } +} + +type matcher struct { + specs.Platform +} + +func (m *matcher) Match(platform specs.Platform) bool { + normalized := Normalize(platform) + return m.OS == normalized.OS && + m.Architecture == normalized.Architecture && + m.Variant == normalized.Variant +} + +func (m *matcher) String() string { + return Format(m.Platform) +} + +// Parse parses the platform specifier syntax into a platform declaration. +// +// Platform specifiers are in the format `<os>|<arch>|<os>/<arch>[/<variant>]`. +// The minimum required information for a platform specifier is the operating +// system or architecture. If there is only a single string (no slashes), the +// value will be matched against the known set of operating systems, then fall +// back to the known set of architectures. The missing component will be +// inferred based on the local environment. +func Parse(specifier string) (specs.Platform, error) { + if strings.Contains(specifier, "*") { + // TODO(stevvooe): need to work out exact wildcard handling + return specs.Platform{}, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: wildcards not yet supported", specifier) + } + + parts := strings.Split(specifier, "/") + + for _, part := range parts { + if !specifierRe.MatchString(part) { + return specs.Platform{}, errors.Wrapf(errdefs.ErrInvalidArgument, "%q is an invalid component of %q: platform specifier component must match %q", part, specifier, specifierRe.String()) + } + } + + var p specs.Platform + switch len(parts) { + case 1: + // in this case, we will test that the value might be an OS, then look + // it up. If it is not known, we'll treat it as an architecture. Since + // we have very little information about the platform here, we are + // going to be a little more strict if we don't know about the argument + // value. + p.OS = normalizeOS(parts[0]) + if isKnownOS(p.OS) { + // picks a default architecture + p.Architecture = runtime.GOARCH + if p.Architecture == "arm" { + // TODO(stevvooe): Resolve arm variant, if not v6 (default) + return specs.Platform{}, errors.Wrapf(errdefs.ErrNotImplemented, "arm support not fully implemented") + } + + return p, nil + } + + p.Architecture, p.Variant = normalizeArch(parts[0], "") + if p.Architecture == "arm" && p.Variant == "v7" { + p.Variant = "" + } + if isKnownArch(p.Architecture) { + p.OS = runtime.GOOS + return p, nil + } + + return specs.Platform{}, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: unknown operating system or architecture", specifier) + case 2: + // In this case, we treat as a regular os/arch pair. We don't care + // about whether or not we know of the platform. + p.OS = normalizeOS(parts[0]) + p.Architecture, p.Variant = normalizeArch(parts[1], "") + if p.Architecture == "arm" && p.Variant == "v7" { + p.Variant = "" + } + + return p, nil + case 3: + // we have a fully specified variant, this is rare + p.OS = normalizeOS(parts[0]) + p.Architecture, p.Variant = normalizeArch(parts[1], parts[2]) + if p.Architecture == "arm64" && p.Variant == "" { + p.Variant = "v8" + } + + return p, nil + } + + return specs.Platform{}, errors.Wrapf(errdefs.ErrInvalidArgument, "%q: cannot parse platform specifier", specifier) +} + +// MustParse is like Parses but panics if the specifier cannot be parsed. +// Simplifies initialization of global variables. +func MustParse(specifier string) specs.Platform { + p, err := Parse(specifier) + if err != nil { + panic("platform: Parse(" + strconv.Quote(specifier) + "): " + err.Error()) + } + return p +} + +// Format returns a string specifier from the provided platform specification. +func Format(platform specs.Platform) string { + if platform.OS == "" { + return "unknown" + } + + return joinNotEmpty(platform.OS, platform.Architecture, platform.Variant) +} + +func joinNotEmpty(s ...string) string { + var ss []string + for _, s := range s { + if s == "" { + continue + } + + ss = append(ss, s) + } + + return strings.Join(ss, "/") +} + +// Normalize validates and translate the platform to the canonical value. +// +// For example, if "Aarch64" is encountered, we change it to "arm64" or if +// "x86_64" is encountered, it becomes "amd64". +func Normalize(platform specs.Platform) specs.Platform { + platform.OS = normalizeOS(platform.OS) + platform.Architecture, platform.Variant = normalizeArch(platform.Architecture, platform.Variant) + + // these fields are deprecated, remove them + platform.OSFeatures = nil + platform.OSVersion = "" + + return platform +} diff --git a/vendor/github.com/containers/buildah/CHANGELOG.md b/vendor/github.com/containers/buildah/CHANGELOG.md index 3bf97a522..8d1944c65 100644 --- a/vendor/github.com/containers/buildah/CHANGELOG.md +++ b/vendor/github.com/containers/buildah/CHANGELOG.md @@ -2,6 +2,22 @@ # Changelog +## v1.14.3 (2020-03-17) + Update containers/storage to v1.16.5 + Bump github.com/containers/storage from 1.16.2 to 1.16.4 + Bump github.com/openshift/imagebuilder from 1.1.1 to 1.1.2 + Update github.com/openshift/imagebuilder vendoring + Update unshare man page to fix script example + Fix compilation errors on non linux platforms + Bump containers/common and opencontainers/selinux versions + Add tests for volume ownership + Preserve volume uid and gid through subsequent commands + Fix FORWARD_NULL errors found by Coverity + Bump github.com/containers/storage from 1.16.1 to 1.16.2 + Fix errors found by codespell + Bump back to v1.15.0-dev + Add Pull Request Template + ## v1.14.2 (2020-03-03) Add Buildah pull request template Bump to containers/storage v1.16.1 diff --git a/vendor/github.com/containers/buildah/CONTRIBUTING.md b/vendor/github.com/containers/buildah/CONTRIBUTING.md index 0178a517c..553cb15dc 100644 --- a/vendor/github.com/containers/buildah/CONTRIBUTING.md +++ b/vendor/github.com/containers/buildah/CONTRIBUTING.md @@ -178,7 +178,7 @@ that has been setup. #### Buildah Mailing List -You can join the Buildah mailing list by sending an email to `buildah-join@lists.buildah.io` with the word `subscribe` in the subject. You can also go to this [page](https://lists.podman.io/admin/lists/buildah.lists.buildah.io/), then scroll down to the bottom of the page and enter your email and optionally name, then click on the "Subscribe" buton. +You can join the Buildah mailing list by sending an email to `buildah-join@lists.buildah.io` with the word `subscribe` in the subject. You can also go to this [page](https://lists.podman.io/admin/lists/buildah.lists.buildah.io/), then scroll down to the bottom of the page and enter your email and optionally name, then click on the "Subscribe" button. #### GitHub You can also use the github diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 6d1d479b1..0fee906e5 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -27,7 +27,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.14.2" + Version = "1.14.3" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/changelog.txt b/vendor/github.com/containers/buildah/changelog.txt index b4c71bf6a..900accf10 100644 --- a/vendor/github.com/containers/buildah/changelog.txt +++ b/vendor/github.com/containers/buildah/changelog.txt @@ -1,3 +1,19 @@ +- Changelog for v1.14.3 (2020-03-17) + * Update containers/storage to v1.16.5 + * Bump github.com/containers/storage from 1.16.2 to 1.16.4 + * Bump github.com/openshift/imagebuilder from 1.1.1 to 1.1.2 + * Update github.com/openshift/imagebuilder vendoring + * Update unshare man page to fix script example + * Fix compilation errors on non linux platforms + * Bump containers/common and opencontainers/selinux versions + * Add tests for volume ownership + * Preserve volume uid and gid through subsequent commands + * Fix FORWARD_NULL errors found by Coverity + * Bump github.com/containers/storage from 1.16.1 to 1.16.2 + * Fix errors found by codespell + * Bump back to v1.15.0-dev + * Add Pull Request Template + - Changelog for v1.14.2 (2020-03-03) * Add Buildah pull request template * Bump to containers/storage v1.16.1 diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index 482fef693..8dfa8aba0 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -656,7 +656,7 @@ func runUsingChrootExecMain() { // Set the hostname. We're already in a distinct UTS namespace and are admins in the user // namespace which created it, so we shouldn't get a permissions error, but seccomp policy // might deny our attempt to call sethostname() anyway, so log a debug message for that. - if options.Spec.Hostname != "" { + if options.Spec != nil && options.Spec.Hostname != "" { if err := unix.Sethostname([]byte(options.Spec.Hostname)); err != nil { logrus.Debugf("failed to set hostname %q for process: %v", options.Spec.Hostname, err) } diff --git a/vendor/github.com/containers/buildah/go.mod b/vendor/github.com/containers/buildah/go.mod index 72fbffe2c..862c7e1a2 100644 --- a/vendor/github.com/containers/buildah/go.mod +++ b/vendor/github.com/containers/buildah/go.mod @@ -4,9 +4,9 @@ go 1.12 require ( github.com/containernetworking/cni v0.7.2-0.20190904153231-83439463f784 - github.com/containers/common v0.4.2 + github.com/containers/common v0.5.0 github.com/containers/image/v5 v5.2.1 - github.com/containers/storage v1.16.1 + github.com/containers/storage v1.16.5 github.com/cyphar/filepath-securejoin v0.2.2 github.com/docker/distribution v2.7.1+incompatible github.com/docker/go-metrics v0.0.1 // indirect @@ -25,9 +25,9 @@ require ( github.com/opencontainers/runc v1.0.0-rc9 github.com/opencontainers/runtime-spec v0.1.2-0.20190618234442-a950415649c7 github.com/opencontainers/runtime-tools v0.9.0 - github.com/opencontainers/selinux v1.3.3 + github.com/opencontainers/selinux v1.4.0 github.com/openshift/api v0.0.0-20200106203948-7ab22a2c8316 - github.com/openshift/imagebuilder v1.1.1 + github.com/openshift/imagebuilder v1.1.2 github.com/pkg/errors v0.9.1 github.com/seccomp/containers-golang v0.0.0-20190312124753-8ca8945ccf5f github.com/seccomp/libseccomp-golang v0.9.1 diff --git a/vendor/github.com/containers/buildah/go.sum b/vendor/github.com/containers/buildah/go.sum index 79dc064ce..17ea81042 100644 --- a/vendor/github.com/containers/buildah/go.sum +++ b/vendor/github.com/containers/buildah/go.sum @@ -103,6 +103,14 @@ github.com/containers/common v0.4.1 h1:Uu7f2ZDM/5xsqOkZwIEVKSjUI3YxKjvNIY5x57kja github.com/containers/common v0.4.1/go.mod h1:m62kenckrWi5rZx32kaLje2Og0hpf6NsaTBn6+b+Oys= github.com/containers/common v0.4.2 h1:O5d1gj/xdpQdZi0MEivRQ/7AeRaVeHdbSP/bvShw458= github.com/containers/common v0.4.2/go.mod h1:m62kenckrWi5rZx32kaLje2Og0hpf6NsaTBn6+b+Oys= +github.com/containers/common v0.4.3 h1:TJ7UQxB8wf//IY4LNZobswrTjbhIjXpidrRbCA2l+kg= +github.com/containers/common v0.4.3/go.mod h1:m62kenckrWi5rZx32kaLje2Og0hpf6NsaTBn6+b+Oys= +github.com/containers/common v0.4.4 h1:oXQUPDQOIQ+XmQ2cWyLCs2TctDfISykAr1gEa3CNwlQ= +github.com/containers/common v0.4.4/go.mod h1:vMkHkvczHslJbUj8xasSQmdNrLUgZYuUxVNGJDfjRIQ= +github.com/containers/common v0.5.0 h1:ZAef7h3oO46PcbTyfooZf8XLHrYad+GkhSu3EhH6P24= +github.com/containers/common v0.5.0/go.mod h1:m62kenckrWi5rZx32kaLje2Og0hpf6NsaTBn6+b+Oys= +github.com/containers/common v1.0.0 h1:sZB48LzGP4bP1CmrkQIFUzdUVBysqRv3kWVk4+qbaVA= +github.com/containers/common v1.0.0/go.mod h1:m62kenckrWi5rZx32kaLje2Og0hpf6NsaTBn6+b+Oys= github.com/containers/conmon v2.0.10+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v4 v4.0.1 h1:idNGHChj0Pyv3vLrxul2oSVMZLeFqpoq3CjLeVgapSQ= github.com/containers/image/v4 v4.0.1/go.mod h1:0ASJH1YgJiX/eqFZObqepgsvIA4XjCgpyfwn9pDGafA= @@ -149,6 +157,14 @@ github.com/containers/storage v1.16.0 h1:sD+s7BmiNBh61CuHN3j8PXGCwMtV9zPVJETAlsh github.com/containers/storage v1.16.0/go.mod h1:nqN09JSi1/RSI1UAUwDYXPRiGSlq5FPbNkN/xb0TfG0= github.com/containers/storage v1.16.1 h1:gVLVqbqaoyopLJbcQ9PQdsnm8SzVy6Vw24fofwMgkE0= github.com/containers/storage v1.16.1/go.mod h1:toFp72SLn/iyJ6YbrnrZ0bW63aH2Qw3dA8JVwL4ADPo= +github.com/containers/storage v1.16.2 h1:S77Y+lmJcnGoPEZB2OOrTrRGyjT8viDCGyhVNNz78h8= +github.com/containers/storage v1.16.2/go.mod h1:/RNmsK01ajCL+VtMSi3W8kHzpBwN+Q5gLYWgfw5wlMg= +github.com/containers/storage v1.16.3 h1:bctiz1I+0TIivtXbrVmy02ZYlOA+IjKIJMzAMTBifj8= +github.com/containers/storage v1.16.3/go.mod h1:dNTv0+BaebIAOGgH34dPtwGPR+Km2fObcfOlFxYFwA0= +github.com/containers/storage v1.16.4 h1:+pEL9A1i11qy1j/MYvh8Y5vs79BBfA+hslyJq1iPOGc= +github.com/containers/storage v1.16.4/go.mod h1:SdysZeLKJOvfHYysUWg9OZUC3gdZWi5b2b7NC18VpPE= +github.com/containers/storage v1.16.5 h1:eHeWEhUEWX3VMIG1Vn1rEjfRoLHUQev3cwtA5zd89wk= +github.com/containers/storage v1.16.5/go.mod h1:SdysZeLKJOvfHYysUWg9OZUC3gdZWi5b2b7NC18VpPE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -281,6 +297,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -369,10 +386,14 @@ github.com/klauspost/compress v1.10.0 h1:92XGj1AcYzA6UrVdd4qIIBrT8OroryvRvdmg/If github.com/klauspost/compress v1.10.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.2 h1:Znfn6hXZAHaLPNnlqUYRrBSReFHYybslgv4PTiyz6P0= github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.2 h1:8d4I0LDiieuGngsqlqOih9ker/NS0LX4V0i+EhiFWg0= +github.com/klauspost/pgzip v1.2.2/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -495,6 +516,8 @@ github.com/opencontainers/selinux v1.3.2 h1:DR4lL9SYVjgcTZKEZIncvDU06fKSc/eygjmN github.com/opencontainers/selinux v1.3.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= github.com/opencontainers/selinux v1.3.3 h1:RX0wAeqtvVSYQcr017X3pFXPkLEtB6V4NjRD7gVQgg4= github.com/opencontainers/selinux v1.3.3/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= +github.com/opencontainers/selinux v1.4.0 h1:cpiX/2wWIju/6My60T6/z9CxNG7c8xTQyEmA9fChpUo= +github.com/opencontainers/selinux v1.4.0/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= github.com/openshift/api v0.0.0-20200106203948-7ab22a2c8316 h1:enQG2QUGwug4fR1yM6hL0Fjzx6Km/exZY6RbSPwMu3o= github.com/openshift/api v0.0.0-20200106203948-7ab22a2c8316/go.mod h1:dv+J0b/HWai0QnMVb37/H0v36klkLBi2TNpPeWDxX10= github.com/openshift/api v3.9.1-0.20190810003144-27fb16909b15+incompatible h1:s55wx8JIG/CKnewev892HifTBrtKzMdvgB3rm4rxC2s= @@ -503,6 +526,8 @@ github.com/openshift/imagebuilder v1.1.0 h1:oT704SkwMEzmIMU/+Uv1Wmvt+p10q3v2WuYM github.com/openshift/imagebuilder v1.1.0/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/openshift/imagebuilder v1.1.1 h1:KAUR31p8UBJdfVO42azWgb+LeMAed2zaKQ19e0C0X2I= github.com/openshift/imagebuilder v1.1.1/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= +github.com/openshift/imagebuilder v1.1.2 h1:vCO8hZQR/4uzo+j0PceBH5aKFcvCDM43UzUGOYQN+Go= +github.com/openshift/imagebuilder v1.1.2/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= diff --git a/vendor/github.com/containers/buildah/image.go b/vendor/github.com/containers/buildah/image.go index d333442b8..94e97d870 100644 --- a/vendor/github.com/containers/buildah/image.go +++ b/vendor/github.com/containers/buildah/image.go @@ -627,7 +627,7 @@ func (i *containerImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, logrus.Debugf("error checking for layer %q in %q: %v", blob.Digest.String(), path, err) } } - if err != nil { + if err != nil || layerFile == nil { logrus.Debugf("error reading layer %q: %v", blob.Digest.String(), err) return nil, -1, errors.Wrapf(err, "error opening file %q to buffer layer blob", filepath.Join(i.path, blob.Digest.String())) } diff --git a/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go b/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go index 6216dac97..8e49395a4 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go +++ b/vendor/github.com/containers/buildah/imagebuildah/stage_executor.go @@ -238,7 +238,13 @@ func (s *StageExecutor) volumeCacheRestore() error { if err := os.Chmod(archivedPath, st.Mode()); err != nil { return errors.Wrapf(err, "error restoring permissions on %q", archivedPath) } - if err := os.Chown(archivedPath, 0, 0); err != nil { + uid := 0 + gid := 0 + if st.Sys() != nil { + uid = util.UID(st) + gid = util.GID(st) + } + if err := os.Chown(archivedPath, uid, gid); err != nil { return errors.Wrapf(err, "error setting ownership on %q", archivedPath) } if err := os.Chtimes(archivedPath, st.ModTime(), st.ModTime()); err != nil { diff --git a/vendor/github.com/containers/buildah/manifests/copy.go b/vendor/github.com/containers/buildah/manifests/copy.go new file mode 100644 index 000000000..7e651a46c --- /dev/null +++ b/vendor/github.com/containers/buildah/manifests/copy.go @@ -0,0 +1,15 @@ +package manifests + +import ( + "github.com/containers/image/v5/signature" +) + +var ( + // storageAllowedPolicyScopes overrides the policy for local storage + // to ensure that we can read images from it. + storageAllowedPolicyScopes = signature.PolicyTransportScopes{ + "": []signature.PolicyRequirement{ + signature.NewPRInsecureAcceptAnything(), + }, + } +) diff --git a/vendor/github.com/containers/buildah/manifests/manifests.go b/vendor/github.com/containers/buildah/manifests/manifests.go new file mode 100644 index 000000000..0fe7e477b --- /dev/null +++ b/vendor/github.com/containers/buildah/manifests/manifests.go @@ -0,0 +1,397 @@ +package manifests + +import ( + "context" + "encoding/json" + stderrors "errors" + "io" + + "github.com/containers/buildah/pkg/manifests" + "github.com/containers/buildah/pkg/supplemented" + cp "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/image" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/signature" + is "github.com/containers/image/v5/storage" + "github.com/containers/image/v5/transports" + "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" + "github.com/containers/storage" + digest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const instancesData = "instances.json" + +// ErrListImageUnknown is returned when we attempt to create an image reference +// for a List that has not yet been saved to an image. +var ErrListImageUnknown = stderrors.New("unable to determine which image holds the manifest list") + +type list struct { + manifests.List + instances map[digest.Digest]string +} + +// List is a manifest list or image index, either created using Create(), or +// loaded from local storage using LoadFromImage(). +type List interface { + manifests.List + SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error) + Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) + Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) + Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) +} + +// PushOptions includes various settings which are needed for pushing the +// manifest list and its instances. +type PushOptions struct { + Store storage.Store + SystemContext *types.SystemContext // github.com/containers/image/types.SystemContext + ImageListSelection cp.ImageListSelection // set to either CopySystemImage, CopyAllImages, or CopySpecificImages + Instances []digest.Digest // instances to copy if ImageListSelection == CopySpecificImages + ReportWriter io.Writer // will be used to log the writing of the list and any blobs + SignBy string // fingerprint of GPG key to use to sign images + RemoveSignatures bool // true to discard signatures in images + ManifestType string // the format to use when saving the list - possible options are oci, v2s1, and v2s2 +} + +// Create creates a new list containing information about the specified image, +// computing its manifest's digest, and retrieving OS and architecture +// information from its configuration blob. Returns the new list, and the +// instanceDigest for the initial image. +func Create() List { + return &list{ + List: manifests.Create(), + instances: make(map[digest.Digest]string), + } +} + +// LoadFromImage reads the manifest list or image index, and additional +// information about where the various instances that it contains live, from an +// image record with the specified ID in local storage. +func LoadFromImage(store storage.Store, image string) (string, List, error) { + img, err := store.Image(image) + if err != nil { + return "", nil, errors.Wrapf(err, "error locating image %q for loading manifest list", image) + } + manifestBytes, err := store.ImageBigData(img.ID, storage.ImageDigestManifestBigDataNamePrefix) + if err != nil { + return "", nil, errors.Wrapf(err, "error locating image %q for loading manifest list", image) + } + manifestList, err := manifests.FromBlob(manifestBytes) + if err != nil { + return "", nil, err + } + list := &list{ + List: manifestList, + instances: make(map[digest.Digest]string), + } + instancesBytes, err := store.ImageBigData(img.ID, instancesData) + if err != nil { + return "", nil, errors.Wrapf(err, "error locating image %q for loading instance list", image) + } + if err := json.Unmarshal(instancesBytes, &list.instances); err != nil { + return "", nil, errors.Wrapf(err, "error decoding instance list for image %q", image) + } + list.instances[""] = img.ID + return img.ID, list, err +} + +// SaveToImage saves the manifest list or image index as the manifest of an +// Image record with the specified names in local storage, generating a random +// image ID if none is specified. It also stores information about where the +// images whose manifests are included in the list can be found. +func (l *list) SaveToImage(store storage.Store, imageID string, names []string, mimeType string) (string, error) { + manifestBytes, err := l.List.Serialize(mimeType) + if err != nil { + return "", err + } + instancesBytes, err := json.Marshal(&l.instances) + if err != nil { + return "", err + } + img, err := store.CreateImage(imageID, names, "", "", &storage.ImageOptions{}) + if err == nil || errors.Cause(err) == storage.ErrDuplicateID { + created := (err == nil) + if created { + imageID = img.ID + l.instances[""] = img.ID + } + err := store.SetImageBigData(imageID, storage.ImageDigestManifestBigDataNamePrefix, manifestBytes, manifest.Digest) + if err != nil { + if created { + if _, err2 := store.DeleteImage(img.ID, true); err2 != nil { + logrus.Errorf("error deleting image %q after failing to save manifest for it", img.ID) + } + } + return "", errors.Wrapf(err, "error saving manifest list to image %q", imageID) + } + err = store.SetImageBigData(imageID, instancesData, instancesBytes, nil) + if err != nil { + if created { + if _, err2 := store.DeleteImage(img.ID, true); err2 != nil { + logrus.Errorf("error deleting image %q after failing to save instance locations for it", img.ID) + } + } + return "", errors.Wrapf(err, "error saving instance list to image %q", imageID) + } + return imageID, nil + } + return "", errors.Wrapf(err, "error creating image to hold manifest list") +} + +// Reference returns an image reference for the composite image being built +// in the list, or an error if the list has never been saved to a local image. +func (l *list) Reference(store storage.Store, multiple cp.ImageListSelection, instances []digest.Digest) (types.ImageReference, error) { + if l.instances[""] == "" { + return nil, errors.Wrap(ErrListImageUnknown, "error building reference to list") + } + s, err := is.Transport.ParseStoreReference(store, l.instances[""]) + if err != nil { + return nil, errors.Wrapf(err, "error creating ImageReference from image %q", l.instances[""]) + } + references := make([]types.ImageReference, 0, len(l.instances)) + whichInstances := make([]digest.Digest, 0, len(l.instances)) + switch multiple { + case cp.CopyAllImages, cp.CopySystemImage: + for instance := range l.instances { + if instance != "" { + whichInstances = append(whichInstances, instance) + } + } + case cp.CopySpecificImages: + for instance := range l.instances { + for _, allowed := range instances { + if instance == allowed { + whichInstances = append(whichInstances, instance) + } + } + } + } + for _, instance := range whichInstances { + imageName := l.instances[instance] + ref, err := alltransports.ParseImageName(imageName) + if err != nil { + return nil, errors.Wrapf(err, "error creating ImageReference from image %q", imageName) + } + references = append(references, ref) + } + return supplemented.Reference(s, references, multiple, instances), nil +} + +// Push saves the manifest list and whichever blobs are needed to a destination location. +func (l *list) Push(ctx context.Context, dest types.ImageReference, options PushOptions) (reference.Canonical, digest.Digest, error) { + // Load the system signing policy. + pushPolicy, err := signature.DefaultPolicy(options.SystemContext) + if err != nil { + return nil, "", errors.Wrapf(err, "error obtaining default signature policy") + } + + // Override the settings for local storage to make sure that we can always read the source "image". + pushPolicy.Transports[is.Transport.Name()] = storageAllowedPolicyScopes + + policyContext, err := signature.NewPolicyContext(pushPolicy) + if err != nil { + return nil, "", errors.Wrapf(err, "error creating new signature policy context") + } + defer func() { + if err2 := policyContext.Destroy(); err2 != nil { + logrus.Errorf("error destroying signature policy context: %v", err2) + } + }() + + // If we were given a media type that corresponds to a multiple-images + // type, reset it to a valid corresponding single-image type, since we + // already expect the image library to infer the list type from the + // image type that we're telling it to force. + singleImageManifestType := options.ManifestType + switch singleImageManifestType { + case v1.MediaTypeImageIndex: + singleImageManifestType = v1.MediaTypeImageManifest + case manifest.DockerV2ListMediaType: + singleImageManifestType = manifest.DockerV2Schema2MediaType + } + + // Build a source reference for our list and grab bag full of blobs. + src, err := l.Reference(options.Store, options.ImageListSelection, options.Instances) + if err != nil { + return nil, "", err + } + copyOptions := &cp.Options{ + ImageListSelection: options.ImageListSelection, + Instances: options.Instances, + SourceCtx: options.SystemContext, + DestinationCtx: options.SystemContext, + ReportWriter: options.ReportWriter, + RemoveSignatures: options.RemoveSignatures, + SignBy: options.SignBy, + ForceManifestMIMEType: singleImageManifestType, + } + + // Copy whatever we were asked to copy. + manifestBytes, err := cp.Image(ctx, policyContext, dest, src, copyOptions) + if err != nil { + return nil, "", err + } + manifestDigest, err := manifest.Digest(manifestBytes) + if err != nil { + return nil, "", err + } + return nil, manifestDigest, nil +} + +// Add adds information about the specified image to the list, computing the +// image's manifest's digest, retrieving OS and architecture information from +// the image's configuration, and recording the image's reference so that it +// can be found at push-time. Returns the instanceDigest for the image. If +// the reference points to an image list, either all instances are added (if +// "all" is true), or the instance which matches "sys" (if "all" is false) will +// be added. +func (l *list) Add(ctx context.Context, sys *types.SystemContext, ref types.ImageReference, all bool) (digest.Digest, error) { + src, err := ref.NewImageSource(ctx, sys) + if err != nil { + return "", errors.Wrapf(err, "error setting up to read manifest and configuration from %q", transports.ImageName(ref)) + } + defer src.Close() + + type instanceInfo struct { + instanceDigest *digest.Digest + OS, Architecture, OSVersion, Variant string + Features, OSFeatures, Annotations []string + Size int64 + } + var instanceInfos []instanceInfo + var manifestDigest digest.Digest + + primaryManifestBytes, primaryManifestType, err := src.GetManifest(ctx, nil) + if err != nil { + return "", errors.Wrapf(err, "error reading manifest from %q", transports.ImageName(ref)) + } + + if manifest.MIMETypeIsMultiImage(primaryManifestType) { + lists, err := manifests.FromBlob(primaryManifestBytes) + if err != nil { + return "", errors.Wrapf(err, "error parsing manifest list in %q", transports.ImageName(ref)) + } + if all { + for i, instance := range lists.OCIv1().Manifests { + platform := instance.Platform + if platform == nil { + platform = &v1.Platform{} + } + instanceDigest := instance.Digest + instanceInfo := instanceInfo{ + instanceDigest: &instanceDigest, + OS: platform.OS, + Architecture: platform.Architecture, + OSVersion: platform.OSVersion, + Variant: platform.Variant, + Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...), + OSFeatures: append([]string{}, platform.OSFeatures...), + Size: instance.Size, + } + instanceInfos = append(instanceInfos, instanceInfo) + } + } else { + list, err := manifest.ListFromBlob(primaryManifestBytes, primaryManifestType) + if err != nil { + return "", errors.Wrapf(err, "error parsing manifest list in %q", transports.ImageName(ref)) + } + instanceDigest, err := list.ChooseInstance(sys) + if err != nil { + return "", errors.Wrapf(err, "error selecting image from manifest list in %q", transports.ImageName(ref)) + } + added := false + for i, instance := range lists.OCIv1().Manifests { + if instance.Digest != instanceDigest { + continue + } + platform := instance.Platform + if platform == nil { + platform = &v1.Platform{} + } + instanceInfo := instanceInfo{ + instanceDigest: &instanceDigest, + OS: platform.OS, + Architecture: platform.Architecture, + OSVersion: platform.OSVersion, + Variant: platform.Variant, + Features: append([]string{}, lists.Docker().Manifests[i].Platform.Features...), + OSFeatures: append([]string{}, platform.OSFeatures...), + Size: instance.Size, + } + instanceInfos = append(instanceInfos, instanceInfo) + added = true + } + if !added { + instanceInfo := instanceInfo{ + instanceDigest: &instanceDigest, + } + instanceInfos = append(instanceInfos, instanceInfo) + } + } + } else { + instanceInfo := instanceInfo{ + instanceDigest: nil, + } + instanceInfos = append(instanceInfos, instanceInfo) + } + + for _, instanceInfo := range instanceInfos { + if instanceInfo.OS == "" || instanceInfo.Architecture == "" { + img, err := image.FromUnparsedImage(ctx, sys, image.UnparsedInstance(src, instanceInfo.instanceDigest)) + if err != nil { + return "", errors.Wrapf(err, "error reading configuration blob from %q", transports.ImageName(ref)) + } + config, err := img.OCIConfig(ctx) + if err != nil { + return "", errors.Wrapf(err, "error reading info about config blob from %q", transports.ImageName(ref)) + } + if instanceInfo.OS == "" { + instanceInfo.OS = config.OS + } + if instanceInfo.Architecture == "" { + instanceInfo.Architecture = config.Architecture + } + } + manifestBytes, manifestType, err := src.GetManifest(ctx, instanceInfo.instanceDigest) + if err != nil { + return "", errors.Wrapf(err, "error reading manifest from %q, instance %q", transports.ImageName(ref), instanceInfo.instanceDigest) + } + if instanceInfo.instanceDigest == nil { + manifestDigest, err = manifest.Digest(manifestBytes) + if err != nil { + return "", errors.Wrapf(err, "error computing digest of manifest from %q", transports.ImageName(ref)) + } + instanceInfo.instanceDigest = &manifestDigest + instanceInfo.Size = int64(len(manifestBytes)) + } else { + if manifestDigest == "" { + manifestDigest = *instanceInfo.instanceDigest + } + } + err = l.List.AddInstance(*instanceInfo.instanceDigest, instanceInfo.Size, manifestType, instanceInfo.OS, instanceInfo.Architecture, instanceInfo.OSVersion, instanceInfo.OSFeatures, instanceInfo.Variant, instanceInfo.Features, instanceInfo.Annotations) + if err != nil { + return "", errors.Wrapf(err, "error adding instance with digest %q", *instanceInfo.instanceDigest) + } + if _, ok := l.instances[*instanceInfo.instanceDigest]; !ok { + l.instances[*instanceInfo.instanceDigest] = transports.ImageName(ref) + } + } + + return manifestDigest, nil +} + +// Remove filters out any instances in the list which match the specified digest. +func (l *list) Remove(instanceDigest digest.Digest) error { + err := l.List.Remove(instanceDigest) + if err == nil { + if _, needToDelete := l.instances[instanceDigest]; needToDelete { + delete(l.instances, instanceDigest) + } + } + return err +} diff --git a/vendor/github.com/containers/buildah/pkg/manifests/errors.go b/vendor/github.com/containers/buildah/pkg/manifests/errors.go new file mode 100644 index 000000000..8398d7efc --- /dev/null +++ b/vendor/github.com/containers/buildah/pkg/manifests/errors.go @@ -0,0 +1,16 @@ +package manifests + +import ( + "errors" +) + +var ( + // ErrDigestNotFound is returned when we look for an image instance + // with a particular digest in a list or index, and fail to find it. + ErrDigestNotFound = errors.New("no image instance matching the specified digest was found in the list or index") + // ErrManifestTypeNotSupported is returned when we attempt to parse a + // manifest with a known MIME type as a list or index, or when we attempt + // to serialize a list or index to a manifest with a MIME type that we + // don't know how to encode. + ErrManifestTypeNotSupported = errors.New("manifest type not supported") +) diff --git a/vendor/github.com/containers/buildah/pkg/manifests/manifests.go b/vendor/github.com/containers/buildah/pkg/manifests/manifests.go new file mode 100644 index 000000000..ea9495ee7 --- /dev/null +++ b/vendor/github.com/containers/buildah/pkg/manifests/manifests.go @@ -0,0 +1,493 @@ +package manifests + +import ( + "encoding/json" + "os" + + "github.com/containers/image/v5/manifest" + digest "github.com/opencontainers/go-digest" + imgspec "github.com/opencontainers/image-spec/specs-go" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// List is a generic interface for manipulating a manifest list or an image +// index. +type List interface { + AddInstance(manifestDigest digest.Digest, manifestSize int64, manifestType, os, architecture, osVersion string, osFeatures []string, variant string, features []string, annotations []string) error + Remove(instanceDigest digest.Digest) error + + SetURLs(instanceDigest digest.Digest, urls []string) error + URLs(instanceDigest digest.Digest) ([]string, error) + + SetAnnotations(instanceDigest *digest.Digest, annotations map[string]string) error + Annotations(instanceDigest *digest.Digest) (map[string]string, error) + + SetOS(instanceDigest digest.Digest, os string) error + OS(instanceDigest digest.Digest) (string, error) + + SetArchitecture(instanceDigest digest.Digest, arch string) error + Architecture(instanceDigest digest.Digest) (string, error) + + SetOSVersion(instanceDigest digest.Digest, osVersion string) error + OSVersion(instanceDigest digest.Digest) (string, error) + + SetVariant(instanceDigest digest.Digest, variant string) error + Variant(instanceDigest digest.Digest) (string, error) + + SetFeatures(instanceDigest digest.Digest, features []string) error + Features(instanceDigest digest.Digest) ([]string, error) + + SetOSFeatures(instanceDigest digest.Digest, osFeatures []string) error + OSFeatures(instanceDigest digest.Digest) ([]string, error) + + Serialize(mimeType string) ([]byte, error) + Instances() []digest.Digest + OCIv1() *v1.Index + Docker() *manifest.Schema2List + + findDocker(instanceDigest digest.Digest) (*manifest.Schema2ManifestDescriptor, error) + findOCIv1(instanceDigest digest.Digest) (*v1.Descriptor, error) +} + +type list struct { + docker manifest.Schema2List + oci v1.Index +} + +// OCIv1 returns the list as a Docker schema 2 list. The returned structure should NOT be modified. +func (l *list) Docker() *manifest.Schema2List { + return &l.docker +} + +// OCIv1 returns the list as an OCI image index. The returned structure should NOT be modified. +func (l *list) OCIv1() *v1.Index { + return &l.oci +} + +// Create creates a new list. +func Create() List { + return &list{ + docker: manifest.Schema2List{ + SchemaVersion: 2, + MediaType: manifest.DockerV2ListMediaType, + }, + oci: v1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + }, + } +} + +// AddInstance adds an entry for the specified manifest digest, with assorted +// additional information specified in parameters, to the list or index. +func (l *list) AddInstance(manifestDigest digest.Digest, manifestSize int64, manifestType, osName, architecture, osVersion string, osFeatures []string, variant string, features []string, annotations []string) error { + if err := l.Remove(manifestDigest); err != nil && !os.IsNotExist(errors.Cause(err)) { + return err + } + + schema2platform := manifest.Schema2PlatformSpec{ + Architecture: architecture, + OS: osName, + OSVersion: osVersion, + OSFeatures: osFeatures, + Variant: variant, + Features: features, + } + l.docker.Manifests = append(l.docker.Manifests, manifest.Schema2ManifestDescriptor{ + Schema2Descriptor: manifest.Schema2Descriptor{ + MediaType: manifestType, + Size: manifestSize, + Digest: manifestDigest, + }, + Platform: schema2platform, + }) + + ociv1platform := v1.Platform{ + Architecture: architecture, + OS: osName, + OSVersion: osVersion, + OSFeatures: osFeatures, + Variant: variant, + } + l.oci.Manifests = append(l.oci.Manifests, v1.Descriptor{ + MediaType: manifestType, + Size: manifestSize, + Digest: manifestDigest, + Platform: &ociv1platform, + }) + + return nil +} + +// Remove filters out any instances in the list which match the specified digest. +func (l *list) Remove(instanceDigest digest.Digest) error { + err := errors.Wrapf(os.ErrNotExist, "no instance matching digest %q found in manifest list", instanceDigest) + newDockerManifests := make([]manifest.Schema2ManifestDescriptor, 0, len(l.docker.Manifests)) + for i := range l.docker.Manifests { + if l.docker.Manifests[i].Digest != instanceDigest { + newDockerManifests = append(newDockerManifests, l.docker.Manifests[i]) + } else { + err = nil + } + } + l.docker.Manifests = newDockerManifests + newOCIv1Manifests := make([]v1.Descriptor, 0, len(l.oci.Manifests)) + for i := range l.oci.Manifests { + if l.oci.Manifests[i].Digest != instanceDigest { + newOCIv1Manifests = append(newOCIv1Manifests, l.oci.Manifests[i]) + } else { + err = nil + } + } + l.oci.Manifests = newOCIv1Manifests + return err +} + +func (l *list) findDocker(instanceDigest digest.Digest) (*manifest.Schema2ManifestDescriptor, error) { + for i := range l.docker.Manifests { + if l.docker.Manifests[i].Digest == instanceDigest { + return &l.docker.Manifests[i], nil + } + } + return nil, errors.Wrapf(ErrDigestNotFound, "no Docker manifest matching digest %q was found in list", instanceDigest.String()) +} + +func (l *list) findOCIv1(instanceDigest digest.Digest) (*v1.Descriptor, error) { + for i := range l.oci.Manifests { + if l.oci.Manifests[i].Digest == instanceDigest { + return &l.oci.Manifests[i], nil + } + } + return nil, errors.Wrapf(ErrDigestNotFound, "no OCI manifest matching digest %q was found in list", instanceDigest.String()) +} + +// SetURLs sets the URLs where the manifest might also be found. +func (l *list) SetURLs(instanceDigest digest.Digest, urls []string) error { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci.URLs = append([]string{}, urls...) + docker.URLs = append([]string{}, urls...) + return nil +} + +// URLs retrieves the locations from which this object might possibly be downloaded. +func (l *list) URLs(instanceDigest digest.Digest) ([]string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return nil, err + } + return append([]string{}, oci.URLs...), nil +} + +// SetAnnotations sets annotations on the image index, or on a specific manifest. +// The field is specific to the OCI image index format, and is not present in Docker manifest lists. +func (l *list) SetAnnotations(instanceDigest *digest.Digest, annotations map[string]string) error { + a := &l.oci.Annotations + if instanceDigest != nil { + oci, err := l.findOCIv1(*instanceDigest) + if err != nil { + return err + } + a = &oci.Annotations + } + (*a) = make(map[string]string) + for k, v := range annotations { + (*a)[k] = v + } + return nil +} + +// Annotations retrieves the annotations which have been set on the image index, or on one instance. +// The field is specific to the OCI image index format, and is not present in Docker manifest lists. +func (l *list) Annotations(instanceDigest *digest.Digest) (map[string]string, error) { + a := l.oci.Annotations + if instanceDigest != nil { + oci, err := l.findOCIv1(*instanceDigest) + if err != nil { + return nil, err + } + a = oci.Annotations + } + annotations := make(map[string]string) + for k, v := range a { + annotations[k] = v + } + return annotations, nil +} + +// SetOS sets the OS field in the platform information associated with the instance with the specified digest. +func (l *list) SetOS(instanceDigest digest.Digest, os string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker.Platform.OS = os + oci.Platform.OS = os + return nil +} + +// OS retrieves the OS field in the platform information associated with the instance with the specified digest. +func (l *list) OS(instanceDigest digest.Digest) (string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return "", err + } + return oci.Platform.OS, nil +} + +// SetArchitecture sets the Architecture field in the platform information associated with the instance with the specified digest. +func (l *list) SetArchitecture(instanceDigest digest.Digest, arch string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker.Platform.Architecture = arch + oci.Platform.Architecture = arch + return nil +} + +// Architecture retrieves the Architecture field in the platform information associated with the instance with the specified digest. +func (l *list) Architecture(instanceDigest digest.Digest) (string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return "", err + } + return oci.Platform.Architecture, nil +} + +// SetOSVersion sets the OSVersion field in the platform information associated with the instance with the specified digest. +func (l *list) SetOSVersion(instanceDigest digest.Digest, osVersion string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker.Platform.OSVersion = osVersion + oci.Platform.OSVersion = osVersion + return nil +} + +// OSVersion retrieves the OSVersion field in the platform information associated with the instance with the specified digest. +func (l *list) OSVersion(instanceDigest digest.Digest) (string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return "", err + } + return oci.Platform.OSVersion, nil +} + +// SetVariant sets the Variant field in the platform information associated with the instance with the specified digest. +func (l *list) SetVariant(instanceDigest digest.Digest, variant string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker.Platform.Variant = variant + oci.Platform.Variant = variant + return nil +} + +// Variant retrieves the Variant field in the platform information associated with the instance with the specified digest. +func (l *list) Variant(instanceDigest digest.Digest) (string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return "", err + } + return oci.Platform.Variant, nil +} + +// SetFeatures sets the features list in the platform information associated with the instance with the specified digest. +// The field is specific to the Docker manifest list format, and is not present in OCI's image indexes. +func (l *list) SetFeatures(instanceDigest digest.Digest, features []string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + docker.Platform.Features = append([]string{}, features...) + // no OCI equivalent + return nil +} + +// Features retrieves the features list from the platform information associated with the instance with the specified digest. +// The field is specific to the Docker manifest list format, and is not present in OCI's image indexes. +func (l *list) Features(instanceDigest digest.Digest) ([]string, error) { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return nil, err + } + return append([]string{}, docker.Platform.Features...), nil +} + +// SetOSFeatures sets the OS features list in the platform information associated with the instance with the specified digest. +func (l *list) SetOSFeatures(instanceDigest digest.Digest, osFeatures []string) error { + docker, err := l.findDocker(instanceDigest) + if err != nil { + return err + } + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return err + } + docker.Platform.OSFeatures = append([]string{}, osFeatures...) + oci.Platform.OSFeatures = append([]string{}, osFeatures...) + return nil +} + +// OSFeatures retrieves the OS features list from the platform information associated with the instance with the specified digest. +func (l *list) OSFeatures(instanceDigest digest.Digest) ([]string, error) { + oci, err := l.findOCIv1(instanceDigest) + if err != nil { + return nil, err + } + return append([]string{}, oci.Platform.OSFeatures...), nil +} + +// FromBlob builds a list from an encoded manifest list or image index. +func FromBlob(manifestBytes []byte) (List, error) { + manifestType := manifest.GuessMIMEType(manifestBytes) + list := &list{ + docker: manifest.Schema2List{ + SchemaVersion: 2, + MediaType: manifest.DockerV2ListMediaType, + }, + oci: v1.Index{ + Versioned: imgspec.Versioned{SchemaVersion: 2}, + }, + } + switch manifestType { + default: + return nil, errors.Wrapf(ErrManifestTypeNotSupported, "unable to load manifest list: unsupported format %q", manifestType) + case manifest.DockerV2ListMediaType: + if err := json.Unmarshal(manifestBytes, &list.docker); err != nil { + return nil, errors.Wrapf(err, "unable to parse Docker manifest list from image") + } + for _, m := range list.docker.Manifests { + list.oci.Manifests = append(list.oci.Manifests, v1.Descriptor{ + MediaType: m.Schema2Descriptor.MediaType, + Size: m.Schema2Descriptor.Size, + Digest: m.Schema2Descriptor.Digest, + Platform: &v1.Platform{ + Architecture: m.Platform.Architecture, + OS: m.Platform.OS, + OSVersion: m.Platform.OSVersion, + OSFeatures: m.Platform.OSFeatures, + Variant: m.Platform.Variant, + }, + }) + } + case v1.MediaTypeImageIndex: + if err := json.Unmarshal(manifestBytes, &list.oci); err != nil { + return nil, errors.Wrapf(err, "unable to parse OCIv1 manifest list") + } + for _, m := range list.oci.Manifests { + platform := m.Platform + if platform == nil { + platform = &v1.Platform{} + } + list.docker.Manifests = append(list.docker.Manifests, manifest.Schema2ManifestDescriptor{ + Schema2Descriptor: manifest.Schema2Descriptor{ + MediaType: m.MediaType, + Size: m.Size, + Digest: m.Digest, + }, + Platform: manifest.Schema2PlatformSpec{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Variant: platform.Variant, + }, + }) + } + } + return list, nil +} + +func (l *list) preferOCI() bool { + // If we have any data that's only in the OCI format, use that. + for _, m := range l.oci.Manifests { + if len(m.URLs) > 0 { + return true + } + if len(m.Annotations) > 0 { + return true + } + } + // If we have any data that's only in the Docker format, use that. + for _, m := range l.docker.Manifests { + if len(m.Platform.Features) > 0 { + return false + } + } + // If we have no manifests, remember that the Docker format is + // explicitly typed, so use that. Otherwise, default to using the OCI + // format. + return len(l.docker.Manifests) != 0 +} + +// Serialize encodes the list using the specified format, or by selecting one +// which it thinks is appropriate. +func (l *list) Serialize(mimeType string) ([]byte, error) { + var manifestBytes []byte + switch mimeType { + case "": + if l.preferOCI() { + manifest, err := json.Marshal(&l.oci) + if err != nil { + return nil, errors.Wrapf(err, "error marshalling OCI image index") + } + manifestBytes = manifest + } else { + manifest, err := json.Marshal(&l.docker) + if err != nil { + return nil, errors.Wrapf(err, "error marshalling Docker manifest list") + } + manifestBytes = manifest + } + case v1.MediaTypeImageIndex: + manifest, err := json.Marshal(&l.oci) + if err != nil { + return nil, errors.Wrapf(err, "error marshalling OCI image index") + } + manifestBytes = manifest + case manifest.DockerV2ListMediaType: + manifest, err := json.Marshal(&l.docker) + if err != nil { + return nil, errors.Wrapf(err, "error marshalling Docker manifest list") + } + manifestBytes = manifest + default: + return nil, errors.Wrapf(ErrManifestTypeNotSupported, "serializing list to type %q not implemented", mimeType) + } + return manifestBytes, nil +} + +// Instances returns the list of image instances mentioned in this list. +func (l *list) Instances() []digest.Digest { + instances := make([]digest.Digest, 0, len(l.oci.Manifests)) + for _, instance := range l.oci.Manifests { + instances = append(instances, instance.Digest) + } + return instances +} diff --git a/vendor/github.com/containers/buildah/pkg/supplemented/errors.go b/vendor/github.com/containers/buildah/pkg/supplemented/errors.go new file mode 100644 index 000000000..6de679b50 --- /dev/null +++ b/vendor/github.com/containers/buildah/pkg/supplemented/errors.go @@ -0,0 +1,17 @@ +package supplemented + +import ( + "errors" + + "github.com/containers/buildah/pkg/manifests" +) + +var ( + // ErrDigestNotFound is returned when we look for an image instance + // with a particular digest in a list or index, and fail to find it. + ErrDigestNotFound = manifests.ErrDigestNotFound + // ErrBlobNotFound is returned when try to figure out which supplemental + // image we should ask for a blob with the specified characteristics, + // based on the information in each of the supplemental images' manifests. + ErrBlobNotFound = errors.New("location of blob could not be determined") +) diff --git a/vendor/github.com/containers/buildah/pkg/supplemented/supplemented.go b/vendor/github.com/containers/buildah/pkg/supplemented/supplemented.go new file mode 100644 index 000000000..5e3c6291a --- /dev/null +++ b/vendor/github.com/containers/buildah/pkg/supplemented/supplemented.go @@ -0,0 +1,400 @@ +package supplemented + +import ( + "container/list" + "context" + "io" + + cp "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/image" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/transports" + "github.com/containers/image/v5/types" + multierror "github.com/hashicorp/go-multierror" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// supplementedImageReference groups multiple references together. +type supplementedImageReference struct { + types.ImageReference + references []types.ImageReference + multiple cp.ImageListSelection + instances []digest.Digest +} + +// supplementedImageSource represents an image, plus all of the blobs of other images. +type supplementedImageSource struct { + types.ImageSource + reference types.ImageReference + manifest []byte // The manifest list or image index. + manifestType string // The MIME type of the manifest list or image index. + sourceDefaultInstances map[types.ImageSource]digest.Digest // The default manifest instances of open ImageSource objects. + sourceInstancesByInstance map[digest.Digest]types.ImageSource // A map from manifest instance digests to open ImageSource objects. + instancesByBlobDigest map[digest.Digest]digest.Digest // A map from blob digests to manifest instance digests. +} + +// Reference groups one reference and some number of additional references +// together as a group. The first reference's default instance will be treated +// as the default instance of the resulting reference, with the other +// references' instances made available as instances for their respective +// digests. +func Reference(ref types.ImageReference, supplemental []types.ImageReference, multiple cp.ImageListSelection, instances []digest.Digest) types.ImageReference { + if len(instances) > 0 { + i := make([]digest.Digest, len(instances)) + copy(i, instances) + instances = i + } + return &supplementedImageReference{ + ImageReference: ref, + references: append([]types.ImageReference{}, supplemental...), + multiple: multiple, + instances: instances, + } +} + +// NewImage returns a new higher-level view of the image. +func (s *supplementedImageReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { + src, err := s.NewImageSource(ctx, sys) + if err != nil { + return nil, errors.Wrapf(err, "error building a new Image using an ImageSource") + } + return image.FromSource(ctx, sys, src) +} + +// NewImageSource opens the referenced images, scans their manifests for +// instances, and builds mappings from each blob mentioned in them to their +// instances. +func (s *supplementedImageReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (iss types.ImageSource, err error) { + sources := make(map[digest.Digest]types.ImageSource) + defaultInstances := make(map[types.ImageSource]digest.Digest) + instances := make(map[digest.Digest]digest.Digest) + var sis *supplementedImageSource + + // Open the default instance for reading. + top, err := s.ImageReference.NewImageSource(ctx, sys) + if err != nil { + return nil, errors.Wrapf(err, "error opening %q as image source", transports.ImageName(s.ImageReference)) + } + + defer func() { + if err != nil { + if iss != nil { + // The composite source has been created. Use its Close method. + if err2 := iss.Close(); err2 != nil { + logrus.Errorf("error opening image: %v", err2) + } + } else if top != nil { + // The composite source has not been created, but the top was already opened. Close it. + if err2 := top.Close(); err2 != nil { + logrus.Errorf("error opening image: %v", err2) + } + } + } + }() + + var addSingle, addMulti func(manifestBytes []byte, manifestType string, src types.ImageSource) error + type manifestToRead struct { + src types.ImageSource + instance *digest.Digest + } + manifestsToRead := list.New() + + addSingle = func(manifestBytes []byte, manifestType string, src types.ImageSource) error { + // Mark this instance as being associated with this ImageSource. + manifestDigest, err := manifest.Digest(manifestBytes) + if err != nil { + return errors.Wrapf(err, "error computing digest over manifest %q", string(manifestBytes)) + } + sources[manifestDigest] = src + + // Parse the manifest as a single image. + man, err := manifest.FromBlob(manifestBytes, manifestType) + if err != nil { + return errors.Wrapf(err, "error parsing manifest %q", string(manifestBytes)) + } + + // Log the config blob's digest and the blobs of its layers as associated with this manifest. + config := man.ConfigInfo() + if config.Digest != "" { + instances[config.Digest] = manifestDigest + logrus.Debugf("blob %q belongs to %q", config.Digest, manifestDigest) + } + + layers := man.LayerInfos() + for _, layer := range layers { + instances[layer.Digest] = manifestDigest + logrus.Debugf("layer %q belongs to %q", layer.Digest, manifestDigest) + } + + return nil + } + + addMulti = func(manifestBytes []byte, manifestType string, src types.ImageSource) error { + // Mark this instance as being associated with this ImageSource. + manifestDigest, err := manifest.Digest(manifestBytes) + if err != nil { + return errors.Wrapf(err, "error computing manifest digest") + } + sources[manifestDigest] = src + + // Parse the manifest as a list of images. + list, err := manifest.ListFromBlob(manifestBytes, manifestType) + if err != nil { + return errors.Wrapf(err, "error parsing manifest blob %q as a %q", string(manifestBytes), manifestType) + } + + // Figure out which of its instances we want to look at. + var chaseInstances []digest.Digest + switch s.multiple { + case cp.CopySystemImage: + instance, err := list.ChooseInstance(sys) + if err != nil { + return errors.Wrapf(err, "error selecting appropriate instance from list") + } + chaseInstances = []digest.Digest{instance} + case cp.CopySpecificImages: + chaseInstances = s.instances + case cp.CopyAllImages: + chaseInstances = list.Instances() + } + + // Queue these manifest instances for reading from this + // ImageSource later, if we don't stumble across them somewhere + // else first. + for _, instanceIterator := range chaseInstances { + instance := instanceIterator + next := &manifestToRead{ + src: src, + instance: &instance, + } + if src == top { + // Prefer any other source. + manifestsToRead.PushBack(next) + } else { + // Prefer this source over the first ("main") one. + manifestsToRead.PushFront(next) + } + } + return nil + } + + visitedReferences := make(map[types.ImageReference]struct{}) + for i, ref := range append([]types.ImageReference{s.ImageReference}, s.references...) { + if _, visited := visitedReferences[ref]; visited { + continue + } + visitedReferences[ref] = struct{}{} + + // Open this image for reading. + var src types.ImageSource + if ref == s.ImageReference { + src = top + } else { + src, err = ref.NewImageSource(ctx, sys) + if err != nil { + return nil, errors.Wrapf(err, "error opening %q as image source", transports.ImageName(ref)) + } + } + + // Read the default manifest for the image. + manifestBytes, manifestType, err := src.GetManifest(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "error reading default manifest from image %q", transports.ImageName(ref)) + } + + // If this is the first image, mark it as our starting point. + if i == 0 { + sources[""] = src + + sis = &supplementedImageSource{ + ImageSource: top, + reference: s, + manifest: manifestBytes, + manifestType: manifestType, + sourceDefaultInstances: defaultInstances, + sourceInstancesByInstance: sources, + instancesByBlobDigest: instances, + } + iss = sis + } + + // Record the digest of the ImageSource's default instance's manifest. + manifestDigest, err := manifest.Digest(manifestBytes) + if err != nil { + return nil, errors.Wrapf(err, "error computing digest of manifest from image %q", transports.ImageName(ref)) + } + sis.sourceDefaultInstances[src] = manifestDigest + + // If the ImageSource's default manifest is a list, parse each of its instances. + if manifest.MIMETypeIsMultiImage(manifestType) { + if err = addMulti(manifestBytes, manifestType, src); err != nil { + return nil, errors.Wrapf(err, "error adding multi-image %q", transports.ImageName(ref)) + } + } else { + if err = addSingle(manifestBytes, manifestType, src); err != nil { + return nil, errors.Wrapf(err, "error adding single image %q", transports.ImageName(ref)) + } + } + } + + // Parse the rest of the instances. + for manifestsToRead.Front() != nil { + front := manifestsToRead.Front() + value := front.Value + manifestToRead, ok := value.(*manifestToRead) + if !ok { + panic("bug: wrong type looking for *manifestToRead in list?") + } + manifestsToRead.Remove(front) + + // If we already read this manifest, no need to read it again. + if _, alreadyRead := sources[*manifestToRead.instance]; alreadyRead { + continue + } + + // Read the instance's manifest. + manifestBytes, manifestType, err := manifestToRead.src.GetManifest(ctx, manifestToRead.instance) + if err != nil { + // if errors.Cause(err) == storage.ErrImageUnknown || os.IsNotExist(errors.Cause(err)) { + // Trust that we either don't need it, or that it's in another reference. + // continue + // } + return nil, errors.Wrapf(err, "error reading manifest for instance %q", manifestToRead.instance) + } + + if manifest.MIMETypeIsMultiImage(manifestType) { + // Add the list's contents. + if err = addMulti(manifestBytes, manifestType, manifestToRead.src); err != nil { + return nil, errors.Wrapf(err, "error adding single image instance %q", manifestToRead.instance) + } + } else { + // Add the single image's contents. + if err = addSingle(manifestBytes, manifestType, manifestToRead.src); err != nil { + return nil, errors.Wrapf(err, "error adding single image instance %q", manifestToRead.instance) + } + } + } + + return iss, nil +} + +func (s *supplementedImageReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { + return errors.Errorf("deletion of images not implemented") +} + +func (s *supplementedImageSource) Close() error { + var returnErr *multierror.Error + closed := make(map[types.ImageSource]struct{}) + for _, sourceInstance := range s.sourceInstancesByInstance { + if _, closed := closed[sourceInstance]; closed { + continue + } + if err := sourceInstance.Close(); err != nil { + returnErr = multierror.Append(returnErr, err) + } + closed[sourceInstance] = struct{}{} + } + if returnErr == nil { + return nil + } + return returnErr.ErrorOrNil() +} + +func (s *supplementedImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { + requestInstanceDigest := instanceDigest + if instanceDigest == nil { + return s.manifest, s.manifestType, nil + } + if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok { + if *instanceDigest == s.sourceDefaultInstances[sourceInstance] { + requestInstanceDigest = nil + } + return sourceInstance.GetManifest(ctx, requestInstanceDigest) + } + return nil, "", errors.Wrapf(ErrDigestNotFound, "error getting manifest for digest %q", *instanceDigest) +} + +func (s *supplementedImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, bic types.BlobInfoCache) (io.ReadCloser, int64, error) { + sourceInstance, ok := s.instancesByBlobDigest[blob.Digest] + if !ok { + return nil, -1, errors.Wrapf(ErrBlobNotFound, "error blob %q in known instances", blob.Digest) + } + src, ok := s.sourceInstancesByInstance[sourceInstance] + if !ok { + return nil, -1, errors.Wrapf(ErrDigestNotFound, "error getting image source for instance %q", sourceInstance) + } + return src.GetBlob(ctx, blob, bic) +} + +func (s *supplementedImageSource) HasThreadSafeGetBlob() bool { + checked := make(map[types.ImageSource]struct{}) + for _, sourceInstance := range s.sourceInstancesByInstance { + if _, checked := checked[sourceInstance]; checked { + continue + } + if !sourceInstance.HasThreadSafeGetBlob() { + return false + } + checked[sourceInstance] = struct{}{} + } + return true +} + +func (s *supplementedImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + var ( + src types.ImageSource + digest digest.Digest + ) + requestInstanceDigest := instanceDigest + if instanceDigest == nil { + if sourceInstance, ok := s.sourceInstancesByInstance[""]; ok { + src = sourceInstance + } + } else { + digest = *instanceDigest + if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok { + src = sourceInstance + } + if *instanceDigest == s.sourceDefaultInstances[src] { + requestInstanceDigest = nil + } + } + if src != nil { + return src.GetSignatures(ctx, requestInstanceDigest) + } + return nil, errors.Wrapf(ErrDigestNotFound, "error finding instance for instance digest %q to read signatures", digest) +} + +func (s *supplementedImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + var src types.ImageSource + requestInstanceDigest := instanceDigest + if instanceDigest == nil { + if sourceInstance, ok := s.sourceInstancesByInstance[""]; ok { + src = sourceInstance + } + } else { + if sourceInstance, ok := s.sourceInstancesByInstance[*instanceDigest]; ok { + src = sourceInstance + } + if *instanceDigest == s.sourceDefaultInstances[src] { + requestInstanceDigest = nil + } + } + if src != nil { + blobInfos, err := src.LayerInfosForCopy(ctx, requestInstanceDigest) + if err != nil { + return nil, errors.Wrapf(err, "error reading layer infos for copy from instance %q", instanceDigest) + } + var manifestDigest digest.Digest + if instanceDigest != nil { + manifestDigest = *instanceDigest + } + for _, blobInfo := range blobInfos { + s.instancesByBlobDigest[blobInfo.Digest] = manifestDigest + } + return blobInfos, nil + } + return nil, errors.Wrapf(ErrDigestNotFound, "error finding instance for instance digest %q to copy layers", *instanceDigest) +} diff --git a/vendor/github.com/containers/buildah/pull.go b/vendor/github.com/containers/buildah/pull.go index 4a38abeab..cbb98cbcf 100644 --- a/vendor/github.com/containers/buildah/pull.go +++ b/vendor/github.com/containers/buildah/pull.go @@ -216,8 +216,13 @@ func Pull(ctx context.Context, imageName string, options PullOptions) (imageID s } else { imageID = img.ID } + if errs == nil { + err = nil + } else { + err = errs.ErrorOrNil() + } - return imageID, errs.ErrorOrNil() + return imageID, err } func pullImage(ctx context.Context, store storage.Store, srcRef types.ImageReference, options PullOptions, sc *types.SystemContext) (types.ImageReference, error) { diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 6df6ef41a..6e4d31d78 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -702,7 +702,9 @@ func runUsingRuntime(isolation Isolation, options RunOptions, configureNetwork b return 1, errors.Wrapf(err, "error creating pipe for notifying to stop stdio") } finishedCopy := make(chan struct{}) + var pargs []string if spec.Process != nil { + pargs = spec.Process.Args if spec.Process.Terminal { copyConsole = true // Create a listening socket for accepting the container's terminal's PTY master. @@ -773,7 +775,7 @@ func runUsingRuntime(isolation Isolation, options RunOptions, configureNetwork b logrus.Debugf("Running %q", create.Args) err = create.Run() if err != nil { - return 1, errors.Wrapf(err, "error creating container for %v: %s", spec.Process.Args, runCollectOutput(errorFds, closeBeforeReadingErrorFds)) + return 1, errors.Wrapf(err, "error creating container for %v: %s", pargs, runCollectOutput(errorFds, closeBeforeReadingErrorFds)) } defer func() { err2 := del.Run() @@ -808,7 +810,7 @@ func runUsingRuntime(isolation Isolation, options RunOptions, configureNetwork b }() if configureNetwork { - teardown, err := runConfigureNetwork(isolation, options, configureNetworks, pid, containerName, spec.Process.Args) + teardown, err := runConfigureNetwork(isolation, options, configureNetworks, pid, containerName, pargs) if teardown != nil { defer teardown() } @@ -1044,6 +1046,9 @@ func runConfigureNetwork(isolation Isolation, options RunOptions, configureNetwo } continue } + if nc.Network == nil { + continue + } cl, err := libcni.ConfListFromConf(nc) if err != nil { return nil, errors.Wrapf(err, "error converting networking configuration from file %q for %v", file, command) @@ -1450,8 +1455,13 @@ func runUsingRuntimeMain() { if err := setChildProcess(); err != nil { os.Exit(1) } + var ospec *specs.Spec + if options.Spec != nil { + ospec = options.Spec + } + // Run the container, start to finish. - status, err := runUsingRuntime(options.Isolation, options.Options, options.ConfigureNetwork, options.ConfigureNetworks, options.MoreCreateArgs, options.Spec, options.BundlePath, options.ContainerName) + status, err := runUsingRuntime(options.Isolation, options.Options, options.ConfigureNetwork, options.ConfigureNetworks, options.MoreCreateArgs, ospec, options.BundlePath, options.ContainerName) if err != nil { fmt.Fprintf(os.Stderr, "error running container: %v\n", err) os.Exit(1) diff --git a/vendor/github.com/containers/buildah/util/util.go b/vendor/github.com/containers/buildah/util/util.go index 05d661b58..a358b7c54 100644 --- a/vendor/github.com/containers/buildah/util/util.go +++ b/vendor/github.com/containers/buildah/util/util.go @@ -365,7 +365,7 @@ func GetHostIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, // GetHostRootIDs uses ID mappings in spec to compute the host-level IDs that will // correspond to UID/GID 0/0 in the container. func GetHostRootIDs(spec *specs.Spec) (uint32, uint32, error) { - if spec.Linux == nil { + if spec == nil || spec.Linux == nil { return 0, 0, nil } return GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, 0, 0) @@ -455,7 +455,7 @@ func FindLocalRuntime(runtime string) string { logrus.Debugf("Error loading container config when searching for local runtime.") return localRuntime } - for _, val := range conf.Libpod.OCIRuntimes[runtime] { + for _, val := range conf.Engine.OCIRuntimes[runtime] { if fileExistsAndNotADir(val) { localRuntime = val break diff --git a/vendor/github.com/containers/buildah/util/util_linux.go b/vendor/github.com/containers/buildah/util/util_linux.go index cca1f9e7e..1a13699df 100644 --- a/vendor/github.com/containers/buildah/util/util_linux.go +++ b/vendor/github.com/containers/buildah/util/util_linux.go @@ -1,6 +1,7 @@ package util import ( + "os" "syscall" "golang.org/x/sys/unix" @@ -18,3 +19,11 @@ func IsCgroup2UnifiedMode() (bool, error) { }) return isUnified, isUnifiedErr } + +func UID(st os.FileInfo) int { + return int(st.Sys().(*syscall.Stat_t).Uid) +} + +func GID(st os.FileInfo) int { + return int(st.Sys().(*syscall.Stat_t).Gid) +} diff --git a/vendor/github.com/containers/buildah/util/util_unsupported.go b/vendor/github.com/containers/buildah/util/util_unsupported.go index 05a68f60b..8810536a6 100644 --- a/vendor/github.com/containers/buildah/util/util_unsupported.go +++ b/vendor/github.com/containers/buildah/util/util_unsupported.go @@ -2,7 +2,19 @@ package util +import ( + "os" +) + // IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode. func IsCgroup2UnifiedMode() (bool, error) { return false, nil } + +func UID(st os.FileInfo) int { + return 0 +} + +func GID(st os.FileInfo) int { + return 0 +} diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 4b536b3a5..db70e53a4 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -12,7 +12,6 @@ import ( "github.com/BurntSushi/toml" "github.com/containers/common/pkg/capabilities" "github.com/containers/common/pkg/unshare" - "github.com/containers/storage" units "github.com/docker/go-units" selinux "github.com/opencontainers/selinux/go-selinux" "github.com/pkg/errors" @@ -29,14 +28,14 @@ const ( ) // RuntimeStateStore is a constant indicating which state store implementation -// should be used by libpod +// should be used by engine type RuntimeStateStore int const ( // InvalidStateStore is an invalid state store InvalidStateStore RuntimeStateStore = iota // InMemoryStateStore is an in-memory state that will not persist data - // on containers and pods between libpod instances or after system + // on containers and pods between engine instances or after system // reboot InMemoryStateStore RuntimeStateStore = iota // SQLiteStateStore is a state backed by a SQLite database @@ -46,12 +45,24 @@ const ( BoltDBStateStore RuntimeStateStore = iota ) +// PullPolicy whether to pull new image +type PullPolicy int + +const ( + // PullImageAlways always try to pull new image when create or run + PullImageAlways PullPolicy = iota + // PullImageMissing pulls image if it is not locally + PullImageMissing + // PullImageNever will never pull new image + PullImageNever +) + // Config contains configuration options for container tools type Config struct { // Containers specify settings that configure how containers will run ont the system Containers ContainersConfig `toml:"containers"` - // Libpod specifies how the container engine based on Libpod will run - Libpod LibpodConfig `toml:"libpod"` + // Engine specifies how the container engine based on Engine will run + Engine EngineConfig `toml:"engine"` // Network section defines the configuration of CNI Plugins Network NetworkConfig `toml:"network"` } @@ -149,11 +160,6 @@ type ContainersConfig struct { // ShmSize holds the size of /dev/shm. ShmSize string `toml:"shm_size"` - // SignaturePolicyPath is the path to a signature policy to use for - // validating images. If left empty, the containers/image default signature - // policy will be used. - SignaturePolicyPath string `toml:"_"` - // UTSNS indicates how to create a UTS namespace for the container UTSNS string `toml:"utsns"` @@ -164,8 +170,12 @@ type ContainersConfig struct { UserNSSize int `toml:"userns_size"` } -// LibpodConfig contains configuration options used to set up a libpod runtime -type LibpodConfig struct { +// EngineConfig contains configuration options used to set up a engine runtime +type EngineConfig struct { + // CgroupCheck indicates the configuration has been rewritten after an + // upgrade to Fedora 31 to change the default OCI runtime for cgroupsv2. + CgroupCheck bool `toml:"cgroup_check,omitempty"` + // CGroupManager is the CGroup Manager to use Valid values are "cgroupfs" // and "systemd". CgroupManager string `toml:"cgroup_manager"` @@ -183,7 +193,7 @@ type LibpodConfig struct { //DetachKeys is the sequence of keys used to detach a container. DetachKeys string `toml:"detach_keys"` - // EnablePortReservation determines whether libpod will reserve ports on the + // EnablePortReservation determines whether engine will reserve ports on the // host when they are forwarded to containers. When enabled, when ports are // forwarded to containers, they are held open by conmon as long as the // container is running, ensuring that they cannot be reused by other @@ -220,9 +230,9 @@ type LibpodConfig struct { // LockType is the type of locking to use. LockType string `toml:"lock_type,omitempty"` - // Namespace is the libpod namespace to use. Namespaces are used to create + // Namespace is the engine namespace to use. Namespaces are used to create // scopes to separate containers and pods in the state. When namespace is - // set, libpod will only view containers and pods in the same namespace. All + // set, engine will only view containers and pods in the same namespace. All // containers and pods created will default to the namespace set here. A // namespace of "", the empty string, is equivalent to no namespace, and all // containers and pods will be visible. The default namespace is "". @@ -244,6 +254,15 @@ type LibpodConfig struct { // OCIRuntimes are the set of configured OCI runtimes (default is runc). OCIRuntimes map[string][]string `toml:"runtimes"` + // PullPolicy determines whether to pull image before creating or running a container + // default is "missing" + PullPolicy string `toml:"pull_policy"` + // RuntimePath is the path to OCI runtime binary for launching containers. + // The first path pointing to a valid file will be used This is used only + // when there are no OCIRuntime/OCIRuntimes defined. It is used only to be + // backward compatible with older versions of Podman. + RuntimePath []string `toml:"runtime_path,omitempty"` + // RuntimeSupportsJSON is the list of the OCI runtimes that support // --format=json. RuntimeSupportsJSON []string `toml:"runtime_supports_json"` @@ -253,12 +272,17 @@ type LibpodConfig struct { RuntimeSupportsNoCgroups []string `toml:"runtime_supports_nocgroups"` // SetOptions contains a subset of config options. It's used to indicate if - // a given option has either been set by the user or by a parsed libpod + // a given option has either been set by the user or by the parsed // configuration file. If not, the corresponding option might be // overwritten by values from the database. This behavior guarantees // backwards compat with older version of libpod and Podman. SetOptions + // SignaturePolicyPath is the path to a signature policy to use for + // validating images. If left empty, the containers/image default signature + // policy will be used. + SignaturePolicyPath string `toml:"_"` + // SDNotify tells container engine to allow containers to notify the host systemd of // readiness using the SD_NOTIFY mechanism. SDNotify bool @@ -278,11 +302,6 @@ type LibpodConfig struct { // before sending kill signal. StopTimeout uint `toml:"stop_timeout"` - // StorageConfig is the configuration used by containers/storage Not - // included in the on-disk config, use the dedicated containers/storage - // configuration file instead. - StorageConfig storage.StoreOptions `toml:"-"` - // TmpDir is the path to a temporary directory to store per-boot container // files. Must be stored in a tmpfs. TmpDir string `toml:"tmp_dir"` @@ -294,7 +313,7 @@ type LibpodConfig struct { } // SetOptions contains a subset of options in a Config. It's used to indicate if -// a given option has either been set by the user or by a parsed libpod +// a given option has either been set by the user or by a parsed engine // configuration file. If not, the corresponding option might be overwritten by // values from the database. This behavior guarantees backwards compat with // older version of libpod and Podman. @@ -364,16 +383,9 @@ func NewConfig(userConfigPath string) (*Config, error) { return nil, err } - // If the caller specified a config path to use, then we read this - // rather then using the system defaults. - if userConfigPath != "" { - var err error - // readConfigFromFile reads in container config in the specified - // file and then merge changes with the current default. - config, err = readConfigFromFile(userConfigPath, config) - if err != nil { - return nil, errors.Wrapf(err, "error reading user config %q", userConfigPath) - } + // read libpod.conf and convert the config to *Config + if err = newLibpodConfig(config); err != nil && !os.IsNotExist(err) { + logrus.Errorf("error reading libpod.conf: %v", err) } // Now, gather the system configs and merge them as needed. @@ -392,7 +404,18 @@ func NewConfig(userConfigPath string) (*Config, error) { logrus.Debugf("Merged system config %q: %v", path, config) } - config.checkCgroupsAndAdjustConfig() + // If the caller specified a config path to use, then we read it to + // override the system defaults. + if userConfigPath != "" { + var err error + // readConfigFromFile reads in container config in the specified + // file and then merge changes with the current default. + config, err = readConfigFromFile(userConfigPath, config) + if err != nil { + return nil, errors.Wrapf(err, "error reading user config %q", userConfigPath) + } + logrus.Debugf("Merged user config %q: %v", userConfigPath, config) + } config.addCAPPrefix() if err := config.Validate(); err != nil { @@ -412,14 +435,14 @@ func readConfigFromFile(path string, config *Config) (*Config, error) { if err != nil { return nil, fmt.Errorf("unable to decode configuration %v: %v", path, err) } - if config.Libpod.VolumePath != "" { - config.Libpod.VolumePathSet = true + if config.Engine.VolumePath != "" { + config.Engine.VolumePathSet = true } - if config.Libpod.StaticDir != "" { - config.Libpod.StaticDirSet = true + if config.Engine.StaticDir != "" { + config.Engine.StaticDirSet = true } - if config.Libpod.TmpDir != "" { - config.Libpod.TmpDirSet = true + if config.Engine.TmpDir != "" { + config.Engine.TmpDirSet = true } return config, err @@ -455,11 +478,11 @@ func systemConfigs() ([]string, error) { return configs, nil } -// checkCgroupsAndAdjustConfig checks if we're running rootless with the systemd +// CheckCgroupsAndAdjustConfig checks if we're running rootless with the systemd // cgroup manager. In case the user session isn't available, we're switching the // cgroup manager to cgroupfs. Note, this only applies to rootless. -func (c *Config) checkCgroupsAndAdjustConfig() { - if !unshare.IsRootless() || c.Libpod.CgroupManager != SystemdCgroupsManager { +func (c *Config) CheckCgroupsAndAdjustConfig() { + if !unshare.IsRootless() || c.Engine.CgroupManager != SystemdCgroupsManager { return } @@ -475,7 +498,7 @@ func (c *Config) checkCgroupsAndAdjustConfig() { logrus.Warningf("For using systemd, you may need to login using an user session") logrus.Warningf("Alternatively, you can enable lingering with: `loginctl enable-linger %d` (possibly as root)", unshare.GetRootlessUID()) logrus.Warningf("Falling back to --cgroup-manager=cgroupfs") - c.Libpod.CgroupManager = CgroupfsCgroupsManager + c.Engine.CgroupManager = CgroupfsCgroupsManager } } @@ -495,32 +518,47 @@ func (c *Config) addCAPPrefix() { func (c *Config) Validate() error { if err := c.Containers.Validate(); err != nil { - return errors.Wrapf(err, "containers config") + return errors.Wrapf(err, " error validating containers config") } if !c.Containers.EnableLabeling { selinux.SetDisabled() } + if err := c.Engine.Validate(); err != nil { + return errors.Wrapf(err, "error validating engine configs") + } + + if err := c.Network.Validate(); err != nil { + return errors.Wrapf(err, "error validating network configs") + } + return nil } -// Validate is the main entry point for Libpod configuration validation +// Validate is the main entry point for Engine configuration validation // It returns an `error` on validation failure, otherwise // `nil`. -func (c *LibpodConfig) Validate() error { +func (c *EngineConfig) Validate() error { // Relative paths can cause nasty bugs, because core paths we use could // shift between runs (or even parts of the program - the OCI runtime // uses a different working directory than we do, for example. - if !filepath.IsAbs(c.StaticDir) { + if c.StaticDir != "" && !filepath.IsAbs(c.StaticDir) { return fmt.Errorf("static directory must be an absolute path - instead got %q", c.StaticDir) } - if !filepath.IsAbs(c.TmpDir) { + if c.TmpDir != "" && !filepath.IsAbs(c.TmpDir) { return fmt.Errorf("temporary directory must be an absolute path - instead got %q", c.TmpDir) } - if !filepath.IsAbs(c.VolumePath) { + if c.VolumePath != "" && !filepath.IsAbs(c.VolumePath) { return fmt.Errorf("volume path must be an absolute path - instead got %q", c.VolumePath) } + + // Check if the pullPolicy from containers.conf is valid + // if it is invalid returns the error + pullPolicy := strings.ToLower(c.PullPolicy) + if _, err := ValidatePullPolicy(pullPolicy); err != nil { + return errors.Wrapf(err, "invalid pull type from containers.conf %q", c.PullPolicy) + } return nil } @@ -583,69 +621,21 @@ func (c *NetworkConfig) Validate() error { return errors.Errorf("invalid cni_plugin_dirs: %s", strings.Join(c.CNIPluginDirs, ",")) } -// DBConfig is a set of Libpod runtime configuration settings that are saved in -// a State when it is first created, and can subsequently be retrieved. -type DBConfig struct { - LibpodRoot string - LibpodTmp string - StorageRoot string - StorageTmp string - GraphDriver string - VolumePath string -} - -// MergeDBConfig merges the configuration from the database. -func (c *Config) MergeDBConfig(dbConfig *DBConfig) error { - - if !c.Libpod.StorageConfigRunRootSet && dbConfig.StorageTmp != "" { - if c.Libpod.StorageConfig.RunRoot != dbConfig.StorageTmp && - c.Libpod.StorageConfig.RunRoot != "" { - logrus.Debugf("Overriding run root %q with %q from database", - c.Libpod.StorageConfig.RunRoot, dbConfig.StorageTmp) - } - c.Libpod.StorageConfig.RunRoot = dbConfig.StorageTmp - } - - if !c.Libpod.StorageConfigGraphRootSet && dbConfig.StorageRoot != "" { - if c.Libpod.StorageConfig.GraphRoot != dbConfig.StorageRoot && - c.Libpod.StorageConfig.GraphRoot != "" { - logrus.Debugf("Overriding graph root %q with %q from database", - c.Libpod.StorageConfig.GraphRoot, dbConfig.StorageRoot) - } - c.Libpod.StorageConfig.GraphRoot = dbConfig.StorageRoot - } - - if !c.Libpod.StorageConfigGraphDriverNameSet && dbConfig.GraphDriver != "" { - if c.Libpod.StorageConfig.GraphDriverName != dbConfig.GraphDriver && - c.Libpod.StorageConfig.GraphDriverName != "" { - logrus.Errorf("User-selected graph driver %q overwritten by graph driver %q from database - delete libpod local files to resolve", - c.Libpod.StorageConfig.GraphDriverName, dbConfig.GraphDriver) - } - c.Libpod.StorageConfig.GraphDriverName = dbConfig.GraphDriver - } - - if !c.Libpod.StaticDirSet && dbConfig.LibpodRoot != "" { - if c.Libpod.StaticDir != dbConfig.LibpodRoot && c.Libpod.StaticDir != "" { - logrus.Debugf("Overriding static dir %q with %q from database", c.Libpod.StaticDir, dbConfig.LibpodRoot) - } - c.Libpod.StaticDir = dbConfig.LibpodRoot - } - - if !c.Libpod.TmpDirSet && dbConfig.LibpodTmp != "" { - if c.Libpod.TmpDir != dbConfig.LibpodTmp && c.Libpod.TmpDir != "" { - logrus.Debugf("Overriding tmp dir %q with %q from database", c.Libpod.TmpDir, dbConfig.LibpodTmp) - } - c.Libpod.TmpDir = dbConfig.LibpodTmp - c.Libpod.EventsLogFilePath = filepath.Join(dbConfig.LibpodTmp, "events", "events.log") - } - - if !c.Libpod.VolumePathSet && dbConfig.VolumePath != "" { - if c.Libpod.VolumePath != dbConfig.VolumePath && c.Libpod.VolumePath != "" { - logrus.Debugf("Overriding volume path %q with %q from database", c.Libpod.VolumePath, dbConfig.VolumePath) - } - c.Libpod.VolumePath = dbConfig.VolumePath +// ValidatePullPolicy check if the pullPolicy from CLI is valid and returns the valid enum type +// if the value from CLI or containers.conf is invalid returns the error +func ValidatePullPolicy(pullPolicy string) (PullPolicy, error) { + switch pullPolicy { + case "always": + return PullImageAlways, nil + case "missing": + return PullImageMissing, nil + case "never": + return PullImageNever, nil + case "": + return PullImageMissing, nil + default: + return PullImageMissing, errors.Errorf("invalid pull policy %q", pullPolicy) } - return nil } // FindConmon iterates over (*Config).ConmonPath and returns the path @@ -653,7 +643,7 @@ func (c *Config) MergeDBConfig(dbConfig *DBConfig) error { // to do a path lookup of "conmon". func (c *Config) FindConmon() (string, error) { foundOutdatedConmon := false - for _, path := range c.Libpod.ConmonPath { + for _, path := range c.Engine.ConmonPath { stat, err := os.Stat(path) if err != nil { continue @@ -689,7 +679,7 @@ func (c *Config) FindConmon() (string, error) { return "", errors.Wrapf(ErrInvalidArg, "could not find a working conmon binary (configured options: %v)", - c.Libpod.ConmonPath) + c.Engine.ConmonPath) } // GetDefaultEnv returns the environment variables for the container. @@ -837,6 +827,9 @@ func isDirectory(path string) error { } func rootlessConfigPath() (string, error) { + if configHome := os.Getenv("XDG_CONFIG_HOME"); configHome != "" { + return filepath.Join(configHome, UserOverrideContainersConfig), nil + } home, err := unshare.HomeDir() if err != nil { return "", err diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index 91106b576..b01db5f88 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -1,9 +1,9 @@ # The containers configuration file specifies all of the available configuration -# command-line options/flags for container runtime tools like Podman & Buildah, +# command-line options/flags for container engine tools like Podman & Buildah, # but in a TOML format that can be easily modified and versioned. # Please refer to containers.conf(5) for details of all configuration options. -# Not all container tools implement all of the options. +# Not all container engines implement all of the options. # All of the options have hard coded defaults and these options will override # the built in defaults. Users can then override these options via the command # line. Container engines will read containers.conf files in up to three @@ -30,7 +30,7 @@ # # volumes = [] -# Used to change the name of the default AppArmor profile of container engines. +# Used to change the name of the default AppArmor profile of container engine. # # apparmor_profile = "container-default" @@ -143,7 +143,8 @@ # # ipcns = "private" -# container engines use container separation using MAC(SELinux) labeling. +# Flag tells container engine to whether to use container separation using +# MAC(SELinux)labeling or not. # Flag is ignored on label disabled systems. # # label = true @@ -167,7 +168,7 @@ # # netns = "private" -# Create /etc/hosts for the container. By default, container engines manage +# Create /etc/hosts for the container. By default, container engine manage # /etc/hosts, automatically adding the container's own IP address. # # no_hosts = false @@ -228,7 +229,7 @@ # # network_config_dir = "/etc/cni/net.d/" -[libpod] +[engine] # Cgroup management implementation used for the runtime. # Valid options “systemd” or “cgroupfs” @@ -260,7 +261,7 @@ # # detach_keys = "ctrl-p,ctrl-q" -# Determines whether libpod will reserve ports on the host when they are +# Determines whether engine will reserve ports on the host when they are # forwarded to containers. When enabled, when ports are forwarded to containers, # ports are held open by as long as the container is running, ensuring that # they cannot be reused by other programs on the host. However, this can cause @@ -297,8 +298,8 @@ # # lock_type** = "shm" -# Default libpod namespace -# If libpod is joined to a namespace, it will see only containers and pods +# Default engine namespace +# If engine is joined to a namespace, it will see only containers and pods # that were created in the same namespace, and will create new containers and # pods in that namespace. # The default namespace is "", which corresponds to no namespace. When no @@ -316,7 +317,10 @@ # # num_locks = 2048 -# Directory for persistent libpod files (database, etc) +# Whether to pull new image before running a container +# pull_policy = "missing" + +# Directory for persistent engine files (database, etc) # By default, this will be configured relative to where the containers/storage # stores containers # Uncomment to change location from this default @@ -339,12 +343,12 @@ # runtime = "runc" # List of the OCI runtimes that support --format=json. When json is supported -# libpod will use it for reporting nicer errors. +# engine will use it for reporting nicer errors. # # runtime_supports_json = ["crun", "runc"] # Paths to look for a valid OCI runtime (runc, runv, etc) -[libpod.runtimes] +[engine.runtimes] # runc = [ # "/usr/bin/runc", # "/usr/sbin/runc", @@ -368,7 +372,7 @@ # Number of seconds to wait for container to exit before sending kill signal. #stop_timeout = 10 -# The [libpod.runtimes] table MUST be the last entry in this file. +# The [engine.runtimes] table MUST be the last entry in this file. # (Unless another table is added) # TOML does not provide a way to end a table other than a further table being # defined, so every key hereafter will be part of [runtimes] and not the main diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index 6b83d7703..04c3f9773 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -98,6 +98,8 @@ const ( // DefaultPidsLimit is the default value for maximum number of processes // allowed inside a container DefaultPidsLimit = 2048 + // DefaultPullPolicy pulls the image if it does not exist locally + DefaultPullPolicy = "missing" // DefaultRootlessSignaturePolicyPath is the default value for the // rootless policy.json file. DefaultRootlessSignaturePolicyPath = ".config/containers/policy.json" @@ -116,12 +118,11 @@ const ( // DefaultConfig defines the default values from containers.conf func DefaultConfig() (*Config, error) { - defaultLibpodConfig, err := defaultConfigFromMemory() + defaultEngineConfig, err := defaultConfigFromMemory() if err != nil { return nil, err } - var signaturePolicyPath string netns := "bridge" if unshare.IsRootless() { home, err := unshare.HomeDir() @@ -130,7 +131,7 @@ func DefaultConfig() (*Config, error) { } sigPath := filepath.Join(home, DefaultRootlessSignaturePolicyPath) if _, err := os.Stat(sigPath); err == nil { - signaturePolicyPath = sigPath + defaultEngineConfig.SignaturePolicyPath = sigPath } netns = "slirp4netns" } @@ -152,37 +153,36 @@ func DefaultConfig() (*Config, error) { Env: []string{ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", }, - EnvHost: false, - HTTPProxy: false, - Init: false, - InitPath: "", - IPCNS: "private", - LogDriver: DefaultLogDriver, - LogSizeMax: DefaultLogSizeMax, - NetNS: netns, - NoHosts: false, - PidsLimit: DefaultPidsLimit, - PidNS: "private", - SeccompProfile: SeccompDefaultPath, - ShmSize: DefaultShmSize, - SignaturePolicyPath: signaturePolicyPath, - UTSNS: "private", - UserNS: "private", - UserNSSize: DefaultUserNSSize, + EnvHost: false, + HTTPProxy: false, + Init: false, + InitPath: "", + IPCNS: "private", + LogDriver: DefaultLogDriver, + LogSizeMax: DefaultLogSizeMax, + NetNS: netns, + NoHosts: false, + PidsLimit: DefaultPidsLimit, + PidNS: "private", + SeccompProfile: SeccompDefaultPath, + ShmSize: DefaultShmSize, + UTSNS: "private", + UserNS: "private", + UserNSSize: DefaultUserNSSize, }, Network: NetworkConfig{ DefaultNetwork: "podman", NetworkConfigDir: cniConfigDir, CNIPluginDirs: cniBinDir, }, - Libpod: *defaultLibpodConfig, + Engine: *defaultEngineConfig, }, nil } -// defaultConfigFromMemory returns a default libpod configuration. Note that the +// defaultConfigFromMemory returns a default engine configuration. Note that the // config is different for root and rootless. It also parses the storage.conf. -func defaultConfigFromMemory() (*LibpodConfig, error) { - c := new(LibpodConfig) +func defaultConfigFromMemory() (*EngineConfig, error) { + c := new(EngineConfig) tmp, err := defaultTmpDir() if err != nil { return nil, err @@ -201,7 +201,6 @@ func defaultConfigFromMemory() (*LibpodConfig, error) { } c.StaticDir = filepath.Join(storeOpts.GraphRoot, "libpod") c.VolumePath = filepath.Join(storeOpts.GraphRoot, "volumes") - c.StorageConfig = storeOpts c.HooksDir = DefaultHooksDirs c.ImageDefaultTransport = _defaultTransport @@ -249,6 +248,7 @@ func defaultConfigFromMemory() (*LibpodConfig, error) { "/usr/local/sbin/conmon", "/run/current-system/sw/bin/conmon", } + c.PullPolicy = DefaultPullPolicy c.RuntimeSupportsJSON = []string{ "crun", "runc", diff --git a/vendor/github.com/containers/common/pkg/config/default_linux.go b/vendor/github.com/containers/common/pkg/config/default_linux.go index 43ef5da78..91b73d344 100644 --- a/vendor/github.com/containers/common/pkg/config/default_linux.go +++ b/vendor/github.com/containers/common/pkg/config/default_linux.go @@ -18,7 +18,7 @@ func isCgroup2UnifiedMode() (isUnified bool, isUnifiedErr error) { if err := syscall.Statfs(cgroupRoot, &st); err != nil { isUnified, isUnifiedErr = false, err } else { - isUnified, isUnifiedErr = st.Type == unix.CGROUP2_SUPER_MAGIC, nil + isUnified, isUnifiedErr = int64(st.Type) == int64(unix.CGROUP2_SUPER_MAGIC), nil } return } diff --git a/vendor/github.com/containers/common/pkg/config/libpodConfig.go b/vendor/github.com/containers/common/pkg/config/libpodConfig.go new file mode 100644 index 000000000..333f43815 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/config/libpodConfig.go @@ -0,0 +1,382 @@ +package config + +/* libpodConfig.go contains deprecated functionality and should not be used any longer */ + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/BurntSushi/toml" + "github.com/containers/common/pkg/unshare" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // _rootlessConfigPath is the path to the rootless libpod.conf in $HOME. + _rootlessConfigPath = ".config/containers/libpod.conf" + + // _rootConfigPath is the path to the libpod configuration file + // This file is loaded to replace the builtin default config before + // runtime options (e.g. WithStorageConfig) are applied. + // If it is not present, the builtin default config is used instead + // This path can be overridden when the runtime is created by using + // NewRuntimeFromConfig() instead of NewRuntime(). + _rootConfigPath = _installPrefix + "/share/containers/libpod.conf" + + // _rootOverrideConfigPath is the path to an override for the default libpod + // configuration file. If OverrideConfigPath exists, it will be used in + // place of the configuration file pointed to by ConfigPath. + _rootOverrideConfigPath = _etcDir + "/containers/libpod.conf" +) + +// ConfigFromLibpod contains configuration options used to set up a libpod runtime +type ConfigFromLibpod struct { + // NOTE: when changing this struct, make sure to update (*Config).Merge(). + + // SetOptions contains a subset of config options. It's used to indicate if + // a given option has either been set by the user or by a parsed libpod + // configuration file. If not, the corresponding option might be + // overwritten by values from the database. This behavior guarantees + // backwards compat with older version of libpod and Podman. + SetOptions + + // VolumePath is the default location that named volumes will be created + // under. This convention is followed by the default volume driver, but + // may not be by other drivers. + VolumePath string `toml:"volume_path,omitempty"` + + // ImageDefaultTransport is the default transport method used to fetch + // images. + ImageDefaultTransport string `toml:"image_default_transport,omitempty"` + + // SignaturePolicyPath is the path to a signature policy to use for + // validating images. If left empty, the containers/image default signature + // policy will be used. + SignaturePolicyPath string `toml:"signature_policy_path,omitempty"` + + // OCIRuntime is the OCI runtime to use. + OCIRuntime string `toml:"runtime,omitempty"` + + // OCIRuntimes are the set of configured OCI runtimes (default is runc). + OCIRuntimes map[string][]string `toml:"runtimes,omitempty"` + + // RuntimeSupportsJSON is the list of the OCI runtimes that support + // --format=json. + RuntimeSupportsJSON []string `toml:"runtime_supports_json,omitempty"` + + // RuntimeSupportsNoCgroups is a list of OCI runtimes that support + // running containers without CGroups. + RuntimeSupportsNoCgroups []string `toml:"runtime_supports_nocgroups,omitempty"` + + // RuntimePath is the path to OCI runtime binary for launching containers. + // The first path pointing to a valid file will be used This is used only + // when there are no OCIRuntime/OCIRuntimes defined. It is used only to be + // backward compatible with older versions of Podman. + RuntimePath []string `toml:"runtime_path,omitempty"` + + // ConmonPath is the path to the Conmon binary used for managing containers. + // The first path pointing to a valid file will be used. + ConmonPath []string `toml:"conmon_path,omitempty"` + + // ConmonEnvVars are environment variables to pass to the Conmon binary + // when it is launched. + ConmonEnvVars []string `toml:"conmon_env_vars,omitempty"` + + // CGroupManager is the CGroup Manager to use Valid values are "cgroupfs" + // and "systemd". + CgroupManager string `toml:"cgroup_manager,omitempty"` + + // InitPath is the path to the container-init binary. + InitPath string `toml:"init_path,omitempty"` + + // StaticDir is the path to a persistent directory to store container + // files. + StaticDir string `toml:"static_dir,omitempty"` + + // TmpDir is the path to a temporary directory to store per-boot container + // files. Must be stored in a tmpfs. + TmpDir string `toml:"tmp_dir,omitempty"` + + // MaxLogSize is the maximum size of container logfiles. + MaxLogSize int64 `toml:"max_log_size,omitempty"` + + // NoPivotRoot sets whether to set no-pivot-root in the OCI runtime. + NoPivotRoot bool `toml:"no_pivot_root,omitempty"` + + // CNIConfigDir sets the directory where CNI configuration files are + // stored. + CNIConfigDir string `toml:"cni_config_dir,omitempty"` + + // CNIPluginDir sets a number of directories where the CNI network + // plugins can be located. + CNIPluginDir []string `toml:"cni_plugin_dir,omitempty"` + + // CNIDefaultNetwork is the network name of the default CNI network + // to attach pods to. + CNIDefaultNetwork string `toml:"cni_default_network,omitempty"` + + // HooksDir holds paths to the directories containing hooks + // configuration files. When the same filename is present in in + // multiple directories, the file in the directory listed last in + // this slice takes precedence. + HooksDir []string `toml:"hooks_dir,omitempty"` + + // Namespace is the libpod namespace to use. Namespaces are used to create + // scopes to separate containers and pods in the state. When namespace is + // set, libpod will only view containers and pods in the same namespace. All + // containers and pods created will default to the namespace set here. A + // namespace of "", the empty string, is equivalent to no namespace, and all + // containers and pods will be visible. The default namespace is "". + Namespace string `toml:"namespace,omitempty"` + + // InfraImage is the image a pod infra container will use to manage + // namespaces. + InfraImage string `toml:"infra_image,omitempty"` + + // InfraCommand is the command run to start up a pod infra container. + InfraCommand string `toml:"infra_command,omitempty"` + + // EnablePortReservation determines whether libpod will reserve ports on the + // host when they are forwarded to containers. When enabled, when ports are + // forwarded to containers, they are held open by conmon as long as the + // container is running, ensuring that they cannot be reused by other + // programs on the host. However, this can cause significant memory usage if + // a container has many ports forwarded to it. Disabling this can save + // memory. + EnablePortReservation bool `toml:"enable_port_reservation,omitempty"` + + // EnableLabeling indicates whether libpod will support container labeling. + EnableLabeling bool `toml:"label,omitempty"` + + // NetworkCmdPath is the path to the slirp4netns binary. + NetworkCmdPath string `toml:"network_cmd_path,omitempty"` + + // NumLocks is the number of locks to make available for containers and + // pods. + NumLocks uint32 `toml:"num_locks,omitempty"` + + // LockType is the type of locking to use. + LockType string `toml:"lock_type,omitempty"` + + // EventsLogger determines where events should be logged. + EventsLogger string `toml:"events_logger,omitempty"` + + // EventsLogFilePath is where the events log is stored. + EventsLogFilePath string `toml:"events_logfile_path,omitempty"` + + //DetachKeys is the sequence of keys used to detach a container. + DetachKeys string `toml:"detach_keys,omitempty"` + + // SDNotify tells Libpod to allow containers to notify the host systemd of + // readiness using the SD_NOTIFY mechanism. + SDNotify bool `toml:",omitempty"` + + // CgroupCheck indicates the configuration has been rewritten after an + // upgrade to Fedora 31 to change the default OCI runtime for cgroupsv2. + CgroupCheck bool `toml:"cgroup_check,omitempty"` +} + +// newLibpodConfig creates a new ConfigFromLibpod and converts it to Config. +// Depending if we're running as root or rootless, we then merge the system configuration followed +// by merging the default config (hard-coded default in memory). +// Note that the OCI runtime is hard-set to `crun` if we're running on a system +// with cgroupsv2. Other OCI runtimes are not yet supporting cgroupsv2. This +// might change in the future. +func newLibpodConfig(c *Config) error { + // Start with the default config and interatively merge + // fields in the system configs. + config := c.libpodConfig() + + // Now, check if the user can access system configs and merge them if needed. + configs, err := systemLibpodConfigs() + if err != nil { + return errors.Wrapf(err, "error finding config on system") + } + + for _, path := range configs { + config, err = readLibpodConfigFromFile(path, config) + if err != nil { + return errors.Wrapf(err, "error reading system config %q", path) + } + } + + // Since runc does not currently support cgroupV2 + // Change to default crun on first running of libpod.conf + // TODO Once runc has support for cgroups, this function should be removed. + if !config.CgroupCheck && unshare.IsRootless() { + cgroupsV2, err := isCgroup2UnifiedMode() + if err != nil { + return err + } + if cgroupsV2 { + path, err := exec.LookPath("crun") + if err != nil { + // Can't find crun path so do nothing + logrus.Warnf("Can not find crun package on the host, containers might fail to run on cgroup V2 systems without crun: %q", err) + } else { + config.CgroupCheck = true + config.OCIRuntime = path + } + } + } + + c.libpodToContainersConfig(config) + + return nil +} + +// readConfigFromFile reads the specified config file at `path` and attempts to +// unmarshal its content into a Config. The config param specifies the previous +// default config. If the path, only specifies a few fields in the Toml file +// the defaults from the config parameter will be used for all other fields. +func readLibpodConfigFromFile(path string, config *ConfigFromLibpod) (*ConfigFromLibpod, error) { + logrus.Debugf("Reading configuration file %q", path) + _, err := toml.DecodeFile(path, config) + if err != nil { + return nil, fmt.Errorf("unable to decode configuration %v: %v", path, err) + } + + // For the sake of backwards compat we need to check if the config fields + // with *Set suffix are set in the config. Note that the storage-related + // fields are NOT set in the config here but in the storage.conf OR directly + // by the user. + if config.VolumePath != "" { + config.VolumePathSet = true + } + if config.StaticDir != "" { + config.StaticDirSet = true + } + if config.TmpDir != "" { + config.TmpDirSet = true + } + + return config, err +} + +func systemLibpodConfigs() ([]string, error) { + if unshare.IsRootless() { + path, err := rootlessLibpodConfigPath() + if err != nil { + return nil, err + } + if _, err := os.Stat(path); err == nil { + containersConfPath, err := rootlessConfigPath() + if err != nil { + containersConfPath = filepath.Join("$HOME", UserOverrideContainersConfig) + } + // TODO: Raise to Warnf, when Podman is updated to + // remove libpod.conf by default + logrus.Debugf("Found deprecated file %s, please remove. Use %s to override defaults.\n", path, containersConfPath) + return []string{path}, nil + } + return nil, err + } + + configs := []string{} + if _, err := os.Stat(_rootConfigPath); err == nil { + // TODO: Raise to Warnf, when Podman is updated to + // remove libpod.conf by default + logrus.Debugf("Found deprecated file %s, please remove. Use %s to override defaults.\n", _rootConfigPath, OverrideContainersConfig) + configs = append(configs, _rootConfigPath) + } + if _, err := os.Stat(_rootOverrideConfigPath); err == nil { + // TODO: Raise to Warnf, when Podman is updated to + // remove libpod.conf by default + logrus.Debugf("Found deprecated file %s, please remove. Use %s to override defaults.\n", _rootOverrideConfigPath, OverrideContainersConfig) + configs = append(configs, _rootOverrideConfigPath) + } + return configs, nil +} + +func rootlessLibpodConfigPath() (string, error) { + home, err := unshare.HomeDir() + if err != nil { + return "", err + } + + return filepath.Join(home, _rootlessConfigPath), nil +} + +func (c *Config) libpodConfig() *ConfigFromLibpod { + return &ConfigFromLibpod{ + InitPath: c.Containers.InitPath, + MaxLogSize: c.Containers.LogSizeMax, + EnableLabeling: c.Containers.EnableLabeling, + + SetOptions: c.Engine.SetOptions, + VolumePath: c.Engine.VolumePath, + ImageDefaultTransport: c.Engine.ImageDefaultTransport, + OCIRuntime: c.Engine.OCIRuntime, + OCIRuntimes: c.Engine.OCIRuntimes, + RuntimeSupportsJSON: c.Engine.RuntimeSupportsJSON, + RuntimeSupportsNoCgroups: c.Engine.RuntimeSupportsNoCgroups, + RuntimePath: c.Engine.RuntimePath, + ConmonPath: c.Engine.ConmonPath, + ConmonEnvVars: c.Engine.ConmonEnvVars, + CgroupManager: c.Engine.CgroupManager, + StaticDir: c.Engine.StaticDir, + TmpDir: c.Engine.TmpDir, + NoPivotRoot: c.Engine.NoPivotRoot, + HooksDir: c.Engine.HooksDir, + Namespace: c.Engine.Namespace, + InfraImage: c.Engine.InfraImage, + InfraCommand: c.Engine.InfraCommand, + EnablePortReservation: c.Engine.EnablePortReservation, + NetworkCmdPath: c.Engine.NetworkCmdPath, + NumLocks: c.Engine.NumLocks, + LockType: c.Engine.LockType, + EventsLogger: c.Engine.EventsLogger, + EventsLogFilePath: c.Engine.EventsLogFilePath, + DetachKeys: c.Engine.DetachKeys, + SDNotify: c.Engine.SDNotify, + CgroupCheck: c.Engine.CgroupCheck, + SignaturePolicyPath: c.Engine.SignaturePolicyPath, + + CNIConfigDir: c.Network.NetworkConfigDir, + CNIPluginDir: c.Network.CNIPluginDirs, + CNIDefaultNetwork: c.Network.DefaultNetwork, + } +} + +func (c *Config) libpodToContainersConfig(libpodConf *ConfigFromLibpod) { + + c.Containers.InitPath = libpodConf.InitPath + c.Containers.LogSizeMax = libpodConf.MaxLogSize + c.Containers.EnableLabeling = libpodConf.EnableLabeling + + c.Engine.SignaturePolicyPath = libpodConf.SignaturePolicyPath + c.Engine.SetOptions = libpodConf.SetOptions + c.Engine.VolumePath = libpodConf.VolumePath + c.Engine.ImageDefaultTransport = libpodConf.ImageDefaultTransport + c.Engine.OCIRuntime = libpodConf.OCIRuntime + c.Engine.OCIRuntimes = libpodConf.OCIRuntimes + c.Engine.RuntimeSupportsJSON = libpodConf.RuntimeSupportsJSON + c.Engine.RuntimeSupportsNoCgroups = libpodConf.RuntimeSupportsNoCgroups + c.Engine.RuntimePath = libpodConf.RuntimePath + c.Engine.ConmonPath = libpodConf.ConmonPath + c.Engine.ConmonEnvVars = libpodConf.ConmonEnvVars + c.Engine.CgroupManager = libpodConf.CgroupManager + c.Engine.StaticDir = libpodConf.StaticDir + c.Engine.TmpDir = libpodConf.TmpDir + c.Engine.NoPivotRoot = libpodConf.NoPivotRoot + c.Engine.HooksDir = libpodConf.HooksDir + c.Engine.Namespace = libpodConf.Namespace + c.Engine.InfraImage = libpodConf.InfraImage + c.Engine.InfraCommand = libpodConf.InfraCommand + c.Engine.EnablePortReservation = libpodConf.EnablePortReservation + c.Engine.NetworkCmdPath = libpodConf.NetworkCmdPath + c.Engine.NumLocks = libpodConf.NumLocks + c.Engine.LockType = libpodConf.LockType + c.Engine.EventsLogger = libpodConf.EventsLogger + c.Engine.EventsLogFilePath = libpodConf.EventsLogFilePath + c.Engine.DetachKeys = libpodConf.DetachKeys + c.Engine.SDNotify = libpodConf.SDNotify + c.Engine.CgroupCheck = libpodConf.CgroupCheck + + c.Network.NetworkConfigDir = libpodConf.CNIConfigDir + c.Network.CNIPluginDirs = libpodConf.CNIPluginDir + c.Network.DefaultNetwork = libpodConf.CNIDefaultNetwork +} diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index c807441cf..0d92a1028 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.16.3 +1.16.5 diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 84bfc9616..05c1450c0 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -5,8 +5,8 @@ require ( github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 github.com/Microsoft/hcsshim v0.8.7 github.com/docker/go-units v0.4.0 - github.com/klauspost/compress v1.10.2 - github.com/klauspost/pgzip v1.2.1 + github.com/klauspost/compress v1.10.3 + github.com/klauspost/pgzip v1.2.2 github.com/mattn/go-shellwords v1.0.10 github.com/mistifyio/go-zfs v2.1.1+incompatible github.com/opencontainers/go-digest v1.0.0-rc1 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 5d86e062f..30183eb00 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -39,8 +39,12 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.2 h1:Znfn6hXZAHaLPNnlqUYRrBSReFHYybslgv4PTiyz6P0= github.com/klauspost/compress v1.10.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.2 h1:8d4I0LDiieuGngsqlqOih9ker/NS0LX4V0i+EhiFWg0= +github.com/klauspost/pgzip v1.2.2/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= diff --git a/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go b/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go index dc2e0c199..a188c510d 100644 --- a/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go +++ b/vendor/github.com/containers/storage/pkg/fileutils/fileutils.go @@ -1,7 +1,6 @@ package fileutils import ( - "errors" "fmt" "io" "os" @@ -10,6 +9,7 @@ import ( "strings" "text/scanner" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -358,6 +358,21 @@ func ReadSymlinkedDirectory(path string) (string, error) { return realPath, nil } +// ReadSymlinkedPath returns the target directory of a symlink. +// The target of the symbolic link can be a file and a directory. +func ReadSymlinkedPath(path string) (realPath string, err error) { + if realPath, err = filepath.Abs(path); err != nil { + return "", errors.Wrapf(err, "unable to get absolute path for %q", path) + } + if realPath, err = filepath.EvalSymlinks(realPath); err != nil { + return "", errors.Wrapf(err, "failed to canonicalise path for %q", path) + } + if _, err := os.Stat(realPath); err != nil { + return "", errors.Wrapf(err, "failed to stat target %q of %q", realPath, path) + } + return realPath, nil +} + // CreateIfNotExists creates a file or a directory only if it does not already exist. func CreateIfNotExists(path string, isDir bool) error { if _, err := os.Stat(path); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/mount/mount.go b/vendor/github.com/containers/storage/pkg/mount/mount.go index 4276d63af..4b888dceb 100644 --- a/vendor/github.com/containers/storage/pkg/mount/mount.go +++ b/vendor/github.com/containers/storage/pkg/mount/mount.go @@ -56,10 +56,11 @@ func Mounted(mountpoint string) (bool, error) { return false, err } - mountpoint, err = fileutils.ReadSymlinkedDirectory(mountpoint) + mountpoint, err = fileutils.ReadSymlinkedPath(mountpoint) if err != nil { return false, err } + // Search the table for the mountpoint for _, e := range entries { if e.Mountpoint == mountpoint { diff --git a/vendor/github.com/containers/storage/pkg/system/process_unix.go b/vendor/github.com/containers/storage/pkg/system/process_unix.go index 26c8b42c1..a9a0dd751 100644 --- a/vendor/github.com/containers/storage/pkg/system/process_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/process_unix.go @@ -20,5 +20,5 @@ func IsProcessAlive(pid int) bool { // KillProcess force-stops a process. func KillProcess(pid int) { - unix.Kill(pid, unix.SIGKILL) + _ = unix.Kill(pid, unix.SIGKILL) } diff --git a/vendor/github.com/containers/storage/pkg/system/rm.go b/vendor/github.com/containers/storage/pkg/system/rm.go index 618c1dc75..510e71428 100644 --- a/vendor/github.com/containers/storage/pkg/system/rm.go +++ b/vendor/github.com/containers/storage/pkg/system/rm.go @@ -7,6 +7,7 @@ import ( "github.com/containers/storage/pkg/mount" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // EnsureRemoveAll wraps `os.RemoveAll` to check for specific errors that can @@ -29,7 +30,9 @@ func EnsureRemoveAll(dir string) error { maxRetry := 100 // Attempt to unmount anything beneath this dir first - mount.RecursiveUnmount(dir) + if err := mount.RecursiveUnmount(dir); err != nil { + logrus.Debugf("RecusiveUnmount on %s failed: %v", dir, err) + } for { err := os.RemoveAll(dir) diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 49699b263..9ff84c666 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -3308,6 +3308,9 @@ const defaultConfigFile = "/etc/containers/storage.conf" // DefaultConfigFile returns the path to the storage config file used func DefaultConfigFile(rootless bool) (string, error) { if rootless { + if configHome := os.Getenv("XDG_CONFIG_HOME"); configHome != "" { + return filepath.Join(configHome, "containers/storage.conf"), nil + } home := homedir.Get() if home == "" { return "", errors.New("cannot determine user's homedir") diff --git a/vendor/github.com/containers/storage/utils.go b/vendor/github.com/containers/storage/utils.go index f1e94fd2b..406032961 100644 --- a/vendor/github.com/containers/storage/utils.go +++ b/vendor/github.com/containers/storage/utils.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/BurntSushi/toml" "github.com/containers/storage/pkg/homedir" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" @@ -158,23 +157,6 @@ func getRootlessStorageOpts(rootlessUID int) (StoreOptions, error) { return opts, nil } -func getTomlStorage(storeOptions *StoreOptions) *tomlConfig { - config := new(tomlConfig) - - config.Storage.Driver = storeOptions.GraphDriverName - config.Storage.RunRoot = storeOptions.RunRoot - config.Storage.GraphRoot = storeOptions.GraphRoot - config.Storage.RootlessStoragePath = storeOptions.RootlessStoragePath - for _, i := range storeOptions.GraphDriverOptions { - s := strings.Split(i, "=") - if s[0] == "overlay.mount_program" { - config.Storage.Options.MountProgram = s[1] - } - } - - return config -} - func getRootlessUID() int { uidEnv := os.Getenv("_CONTAINERS_ROOTLESS_UID") if uidEnv != "" { @@ -244,23 +226,6 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) { rootlessStoragePath = strings.Replace(rootlessStoragePath, "$USER", usr.Username, -1) storageOpts.GraphRoot = rootlessStoragePath } - } else { - if err := os.MkdirAll(filepath.Dir(storageConf), 0755); err != nil { - return storageOpts, errors.Wrapf(err, "cannot make directory %s", filepath.Dir(storageConf)) - } - file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf) - } - - tomlConfiguration := getTomlStorage(&storageOpts) - defer file.Close() - enc := toml.NewEncoder(file) - if err := enc.Encode(tomlConfiguration); err != nil { - os.Remove(storageConf) - - return storageOpts, errors.Wrapf(err, "failed to encode %s", storageConf) - } } } return storageOpts, nil diff --git a/vendor/github.com/fsnotify/fsnotify/.editorconfig b/vendor/github.com/fsnotify/fsnotify/.editorconfig index ba49e3c23..fad895851 100644 --- a/vendor/github.com/fsnotify/fsnotify/.editorconfig +++ b/vendor/github.com/fsnotify/fsnotify/.editorconfig @@ -1,5 +1,12 @@ root = true -[*] +[*.go] indent_style = tab indent_size = 4 +insert_final_newline = true + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/vendor/github.com/fsnotify/fsnotify/.gitattributes b/vendor/github.com/fsnotify/fsnotify/.gitattributes new file mode 100644 index 000000000..32f1001be --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.gitattributes @@ -0,0 +1 @@ +go.sum linguist-generated diff --git a/vendor/github.com/fsnotify/fsnotify/.travis.yml b/vendor/github.com/fsnotify/fsnotify/.travis.yml index 981d1bb81..a9c30165c 100644 --- a/vendor/github.com/fsnotify/fsnotify/.travis.yml +++ b/vendor/github.com/fsnotify/fsnotify/.travis.yml @@ -2,29 +2,35 @@ sudo: false language: go go: - - 1.8.x - - 1.9.x - - tip + - "stable" + - "1.11.x" + - "1.10.x" + - "1.9.x" matrix: + include: + - go: "stable" + env: GOLINT=true allow_failures: - go: tip fast_finish: true -before_script: - - go get -u github.com/golang/lint/golint + +before_install: + - if [ ! -z "${GOLINT}" ]; then go get -u golang.org/x/lint/golint; fi script: - - go test -v --race ./... + - go test --race ./... after_script: - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" - - test -z "$(golint ./... | tee /dev/stderr)" + - if [ ! -z "${GOLINT}" ]; then echo running golint; golint --set_exit_status ./...; else echo skipping golint; fi - go vet ./... os: - linux - osx + - windows notifications: email: false diff --git a/vendor/github.com/fsnotify/fsnotify/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE index f21e54080..e180c8fb0 100644 --- a/vendor/github.com/fsnotify/fsnotify/LICENSE +++ b/vendor/github.com/fsnotify/fsnotify/LICENSE @@ -1,5 +1,5 @@ Copyright (c) 2012 The Go Authors. All rights reserved. -Copyright (c) 2012 fsnotify Authors. All rights reserved. +Copyright (c) 2012-2019 fsnotify Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index 399320741..b2629e522 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -10,16 +10,16 @@ go get -u golang.org/x/sys/... Cross platform: Windows, Linux, BSD and macOS. -|Adapter |OS |Status | -|----------|----------|----------| -|inotify |Linux 2.6.27 or later, Android\*|Supported [](https://travis-ci.org/fsnotify/fsnotify)| -|kqueue |BSD, macOS, iOS\*|Supported [](https://travis-ci.org/fsnotify/fsnotify)| -|ReadDirectoryChangesW|Windows|Supported [](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| -|FSEvents |macOS |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| -|FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)| -|fanotify |Linux 2.6.37+ | | -|USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)| -|Polling |*All* |[Maybe](https://github.com/fsnotify/fsnotify/issues/9)| +| Adapter | OS | Status | +| --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| inotify | Linux 2.6.27 or later, Android\* | Supported [](https://travis-ci.org/fsnotify/fsnotify) | +| kqueue | BSD, macOS, iOS\* | Supported [](https://travis-ci.org/fsnotify/fsnotify) | +| ReadDirectoryChangesW | Windows | Supported [](https://travis-ci.org/fsnotify/fsnotify) | +| FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | +| FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/issues/12) | +| fanotify | Linux 2.6.37+ | [Planned](https://github.com/fsnotify/fsnotify/issues/114) | +| USN Journals | Windows | [Maybe](https://github.com/fsnotify/fsnotify/issues/53) | +| Polling | *All* | [Maybe](https://github.com/fsnotify/fsnotify/issues/9) | \* Android and iOS are untested. @@ -33,6 +33,53 @@ All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based o Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. +## Usage + +```go +package main + +import ( + "log" + + "github.com/fsnotify/fsnotify" +) + +func main() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + done := make(chan bool) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + log.Println("event:", event) + if event.Op&fsnotify.Write == fsnotify.Write { + log.Println("modified file:", event.Name) + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("error:", err) + } + } + }() + + err = watcher.Add("/tmp/foo") + if err != nil { + log.Fatal(err) + } + <-done +} +``` + ## Contributing Please refer to [CONTRIBUTING][] before opening an issue or pull request. @@ -65,6 +112,10 @@ There are OS-specific limits as to how many watches can be created: * Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. * BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. +**Why don't notifications work with NFS filesystems or filesystem in userspace (FUSE)?** + +fsnotify requires support from underlying OS to work. The current NFS protocol does not provide network level support for file notifications. + [#62]: https://github.com/howeyc/fsnotify/issues/62 [#18]: https://github.com/fsnotify/fsnotify/issues/18 [#11]: https://github.com/fsnotify/fsnotify/issues/11 diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index 190bf0de5..89cab046d 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -63,4 +63,6 @@ func (e Event) String() string { } // Common errors that can be reported by a watcher -var ErrEventOverflow = errors.New("fsnotify queue overflow") +var ( + ErrEventOverflow = errors.New("fsnotify queue overflow") +) diff --git a/vendor/github.com/fsnotify/fsnotify/go.mod b/vendor/github.com/fsnotify/fsnotify/go.mod new file mode 100644 index 000000000..ff11e13f2 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/go.mod @@ -0,0 +1,5 @@ +module github.com/fsnotify/fsnotify + +go 1.13 + +require golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 diff --git a/vendor/github.com/fsnotify/fsnotify/go.sum b/vendor/github.com/fsnotify/fsnotify/go.sum new file mode 100644 index 000000000..f60af9855 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go index cc7db4b22..b33f2b4d4 100644 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go @@ -40,12 +40,12 @@ func newFdPoller(fd int) (*fdPoller, error) { poller.fd = fd // Create epoll fd - poller.epfd, errno = unix.EpollCreate1(0) + poller.epfd, errno = unix.EpollCreate1(unix.EPOLL_CLOEXEC) if poller.epfd == -1 { return nil, errno } // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK) + errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK|unix.O_CLOEXEC) if errno != nil { return nil, errno } diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go index 7d8de1451..2306c4620 100644 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go @@ -8,4 +8,4 @@ package fsnotify import "golang.org/x/sys/unix" -const openMode = unix.O_NONBLOCK | unix.O_RDONLY +const openMode = unix.O_NONBLOCK | unix.O_RDONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go index 9139e1716..870c4d6d1 100644 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go @@ -9,4 +9,4 @@ package fsnotify import "golang.org/x/sys/unix" // note: this constant is not defined on BSD -const openMode = unix.O_EVTONLY +const openMode = unix.O_EVTONLY | unix.O_CLOEXEC diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 73ac3c630..86553c2c3 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -66,7 +66,7 @@ var ( // A Decoder can be used in two modes: // // 1) As a stream, or -// 2) For stateless decoding using DecodeAll or DecodeBuffer. +// 2) For stateless decoding using DecodeAll. // // Only a single stream can be decoded concurrently, but the same decoder // can run multiple concurrent stateless decodes. It is even possible to diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go new file mode 100644 index 000000000..4375e08b4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -0,0 +1,521 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. +// Based on work by Yann Collet, released under BSD License. + +package zstd + +import "fmt" + +const ( + betterLongTableBits = 19 // Bits used in the long match table + betterLongTableSize = 1 << betterLongTableBits // Size of the table + + // Note: Increasing the short table bits or making the hash shorter + // can actually lead to compression degradation since it will 'steal' more from the + // long match table and match offsets are quite big. + // This greatly depends on the type of input. + betterShortTableBits = 13 // Bits used in the short match table + betterShortTableSize = 1 << betterShortTableBits // Size of the table +) + +type prevEntry struct { + offset int32 + prev int32 +} + +// betterFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. +// The long match table contains the previous entry with the same hash, +// effectively making it a "chain" of length 2. +// When we find a long match we choose between the two values and select the longest. +// When we find a short match, after checking the long, we check if we can find a long at n+1 +// and that it is longer (lazy matching). +type betterFastEncoder struct { + fastBase + table [betterShortTableSize]tableEntry + longTable [betterLongTableSize]prevEntry +} + +// Encode improves compression... +func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 2 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = prevEntry{} + } + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + for i := range e.longTable[:] { + v := e.longTable[i].offset + v2 := e.longTable[i].prev + if v < minOff { + v = 0 + v2 = 0 + } else { + v = v - e.cur + e.maxMatchOff + if v2 < minOff { + v2 = 0 + } else { + v2 = v2 - e.cur + e.maxMatchOff + } + } + e.longTable[i] = prevEntry{ + offset: v, + prev: v2, + } + } + e.cur = e.maxMatchOff + break + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 1. + stepSize := int32(e.o.targetLength) + if stepSize == 0 { + stepSize++ + } + + const kSearchStrength = 9 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + var t int32 + // We allow the encoder to optionally turn off repeat offsets across blocks + canRepeat := len(blk.sequences) > 2 + var matched int32 + + for { + if debugAsserts && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHashS := hash5(cv, betterShortTableBits) + nextHashL := hash8(cv, betterLongTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + const repOff = 1 + repIndex := s - offset1 + repOff + off := s + e.cur + e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset} + e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)} + + if canRepeat { + if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Index match start+1 (long) -> s - 1 + index0 := s + repOff + s += lenght + repOff + + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + // Index skipped... + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + index0 += 2 + } + cv = load6432(src, s) + continue + } + const repOff2 = 1 + + // We deviate from the reference encoder and also check offset 2. + // Still slower and not much better, so disabled. + // repIndex = s - offset2 + repOff2 + if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { + // Consider history as well. + var seq seq + lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 2 + seq.offset = 2 + if debugSequences { + println("repeat sequence 2", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + index0 := s + repOff2 + s += lenght + repOff2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + + // Index skipped... + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + index0 += 2 + } + cv = load6432(src, s) + // Swap offsets + offset1, offset2 = offset2, offset1 + continue + } + } + // Find the offsets of our two matches. + coffsetL := candidateL.offset - e.cur + coffsetLP := candidateL.prev - e.cur + + // Check if we have a long match. + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matched = e.matchlen(s+8, coffsetL+8, src) + 8 + t = coffsetL + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + + if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { + // Found a long match, at least 8 bytes. + prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8 + if prevMatch > matched { + matched = prevMatch + t = coffsetLP + } + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + } + break + } + + // Check if we have a long match on prev. + if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { + // Found a long match, at least 8 bytes. + matched = e.matchlen(s+8, coffsetLP+8, src) + 8 + t = coffsetLP + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + break + } + + coffsetS := candidateS.offset - e.cur + + // Check if we have a short match. + if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { + // found a regular match + matched = e.matchlen(s+4, coffsetS+4, src) + 4 + + // See if we can find a long match at s+1 + const checkAt = 1 + cv := load6432(src, s+checkAt) + nextHashL = hash8(cv, betterLongTableBits) + candidateL = e.longTable[nextHashL] + coffsetL = candidateL.offset - e.cur + + // We can store it, since we have at least a 4 byte match. + e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset} + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 + if matchedNext > matched { + t = coffsetL + s += checkAt + matched = matchedNext + if debugMatches { + println("long match (after short)") + } + break + } + } + + // Check prev long... + coffsetL = candidateL.prev - e.cur + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 + if matchedNext > matched { + t = coffsetL + s += checkAt + matched = matchedNext + if debugMatches { + println("prev long match (after short)") + } + break + } + } + t = coffsetS + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugAsserts && t < 0 { + panic("t<0") + } + if debugMatches { + println("short match") + } + break + } + + // No match found, move forward in input. + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + + if debugAsserts && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the n-byte match as long as possible. + l := matched + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) -> s - 1 + index0 := s - l + 1 + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + index0 += 2 + } + + cv = load6432(src, s) + if !canRepeat { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash5(cv, betterShortTableBits) + nextHashL := hash8(cv, betterLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} + e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)} + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + +// EncodeNoHist will encode a block with no history and no following blocks. +// Most notable difference is that src will not be copied for history and +// we do not need to check for max match length. +func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + e.Encode(blk, src) +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 0ffea7655..d640e6a9f 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -172,55 +172,6 @@ encodeLoop: cv = load6432(src, s) continue } - const repOff2 = 1 - // We deviate from the reference encoder and also check offset 2. - // Slower and not consistently better, so disabled. - // repIndex = s - offset2 + repOff2 - if false && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff2*8)) { - // Consider history as well. - var seq seq - lenght := 4 + e.matchlen(s+4+repOff2, repIndex+4, src) - - seq.matchLen = uint32(lenght - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := s + repOff2 - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) - - // rep 2 - seq.offset = 2 - if debugSequences { - println("repeat sequence 2", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 - nextEmit = s - if s >= sLimit { - if debug { - println("repeat ended", s, lenght) - - } - break encodeLoop - } - cv = load6432(src, s) - // Swap offsets - offset1, offset2 = offset2, offset1 - continue - } } // Find the offsets of our two matches. coffsetL := s - (candidateL.offset - e.cur) @@ -372,7 +323,7 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv1>>8, dFastShortTableBits) + nextHashS := hash5(cv, dFastShortTableBits) nextHashL := hash8(cv, dFastLongTableBits) // We have at least 4 byte match. diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 28134b158..1387b8082 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -6,6 +6,7 @@ package zstd import ( "fmt" + "math" "math/bits" "github.com/klauspost/compress/zstd/internal/xxhash" @@ -23,7 +24,7 @@ type tableEntry struct { offset int32 } -type fastEncoder struct { +type fastBase struct { o encParams // cur is the offset at the start of hist cur int32 @@ -31,18 +32,22 @@ type fastEncoder struct { maxMatchOff int32 hist []byte crc *xxhash.Digest - table [tableSize]tableEntry tmp [8]byte blk *blockEnc } +type fastEncoder struct { + fastBase + table [tableSize]tableEntry +} + // CRC returns the underlying CRC writer. -func (e *fastEncoder) CRC() *xxhash.Digest { +func (e *fastBase) CRC() *xxhash.Digest { return e.crc } // AppendCRC will append the CRC to the destination slice and return it. -func (e *fastEncoder) AppendCRC(dst []byte) []byte { +func (e *fastBase) AppendCRC(dst []byte) []byte { crc := e.crc.Sum(e.tmp[:0]) dst = append(dst, crc[7], crc[6], crc[5], crc[4]) return dst @@ -50,7 +55,7 @@ func (e *fastEncoder) AppendCRC(dst []byte) []byte { // WindowSize returns the window size of the encoder, // or a window size small enough to contain the input size, if > 0. -func (e *fastEncoder) WindowSize(size int) int32 { +func (e *fastBase) WindowSize(size int) int32 { if size > 0 && size < int(e.maxMatchOff) { b := int32(1) << uint(bits.Len(uint(size))) // Keep minimum window. @@ -63,7 +68,7 @@ func (e *fastEncoder) WindowSize(size int) int32 { } // Block returns the current block. -func (e *fastEncoder) Block() *blockEnc { +func (e *fastBase) Block() *blockEnc { return e.blk } @@ -169,9 +174,22 @@ encodeLoop: if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+6, repIndex+4, src) + var length int32 + // length = 4 + e.matchlen(s+6, repIndex+4, src) + { + a := src[s+6:] + b := src[repIndex+4:] + endI := len(a) & (math.MaxInt32 - 7) + length = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -197,11 +215,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + 2 + s += length + 2 nextEmit = s if s >= sLimit { if debug { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -257,7 +275,20 @@ encodeLoop: } // Extend the 4-byte match as long as possible. - l := e.matchlen(s+4, t+4, src) + 4 + //l := e.matchlen(s+4, t+4, src) + 4 + var l int32 + { + a := src[s+4:] + b := src[t+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } // Extend backwards tMin := s - e.maxMatchOff @@ -294,7 +325,20 @@ encodeLoop: if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { // We have at least 4 byte match. // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) + //l := 4 + e.matchlen(s+4, o2+4, src) + var l int32 + { + a := src[s+4:] + b := src[o2+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } // Store this, since we have it. nextHash := hash6(cv, hashLog) @@ -412,10 +456,23 @@ encodeLoop: if len(blk.sequences) > 2 && load3232(src, repIndex) == uint32(cv>>16) { // Consider history as well. var seq seq - // lenght := 4 + e.matchlen(s+6, repIndex+4, src) - lenght := 4 + int32(matchLen(src[s+6:], src[repIndex+4:])) + // length := 4 + e.matchlen(s+6, repIndex+4, src) + // length := 4 + int32(matchLen(src[s+6:], src[repIndex+4:])) + var length int32 + { + a := src[s+6:] + b := src[repIndex+4:] + endI := len(a) & (math.MaxInt32 - 7) + length = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -441,11 +498,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + 2 + s += length + 2 nextEmit = s if s >= sLimit { if debug { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -498,7 +555,20 @@ encodeLoop: // Extend the 4-byte match as long as possible. //l := e.matchlenNoHist(s+4, t+4, src) + 4 - l := int32(matchLen(src[s+4:], src[t+4:])) + 4 + // l := int32(matchLen(src[s+4:], src[t+4:])) + 4 + var l int32 + { + a := src[s+4:] + b := src[t+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } // Extend backwards tMin := s - e.maxMatchOff @@ -536,7 +606,20 @@ encodeLoop: // We have at least 4 byte match. // No need to check backwards. We come straight from a match //l := 4 + e.matchlenNoHist(s+4, o2+4, src) - l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) + // l := 4 + int32(matchLen(src[s+4:], src[o2+4:])) + var l int32 + { + a := src[s+4:] + b := src[o2+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } // Store this, since we have it. nextHash := hash6(cv, hashLog) @@ -571,7 +654,7 @@ encodeLoop: } } -func (e *fastEncoder) addBlock(src []byte) int32 { +func (e *fastBase) addBlock(src []byte) int32 { if debugAsserts && e.cur > bufferReset { panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset)) } @@ -602,17 +685,17 @@ func (e *fastEncoder) addBlock(src []byte) int32 { // useBlock will replace the block with the provided one, // but transfer recent offsets from the previous. -func (e *fastEncoder) UseBlock(enc *blockEnc) { +func (e *fastBase) UseBlock(enc *blockEnc) { enc.reset(e.blk) e.blk = enc } -func (e *fastEncoder) matchlenNoHist(s, t int32, src []byte) int32 { +func (e *fastBase) matchlenNoHist(s, t int32, src []byte) int32 { // Extend the match to be as long as possible. return int32(matchLen(src[s:], src[t:])) } -func (e *fastEncoder) matchlen(s, t int32, src []byte) int32 { +func (e *fastBase) matchlen(s, t int32, src []byte) int32 { if debugAsserts { if s < 0 { err := fmt.Sprintf("s (%d) < 0", s) @@ -626,18 +709,17 @@ func (e *fastEncoder) matchlen(s, t int32, src []byte) int32 { err := fmt.Sprintf("s (%d) - t (%d) > maxMatchOff (%d)", s, t, e.maxMatchOff) panic(err) } - } - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) + if len(src)-int(s) > maxCompressedBlockSize { + panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) + } } // Extend the match to be as long as possible. - return int32(matchLen(src[s:s1], src[t:])) + return int32(matchLen(src[s:], src[t:])) } // Reset the encoding table. -func (e *fastEncoder) Reset() { +func (e *fastBase) Reset() { if e.blk == nil { e.blk = &blockEnc{} e.blk.init() diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 4032fb9fc..67d45efb9 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -71,15 +71,14 @@ func NewWriter(w io.Writer, opts ...EOption) (*Encoder, error) { } if w != nil { e.Reset(w) - } else { - e.init.Do(func() { - e.initialize() - }) } return &e, nil } func (e *Encoder) initialize() { + if e.o.concurrent == 0 { + e.o.setDefault() + } e.encoders = make(chan encoder, e.o.concurrent) for i := 0; i < e.o.concurrent; i++ { e.encoders <- e.o.encoder() @@ -89,9 +88,6 @@ func (e *Encoder) initialize() { // Reset will re-initialize the writer and new writes will encode to the supplied writer // as a new, independent stream. func (e *Encoder) Reset(w io.Writer) { - e.init.Do(func() { - e.initialize() - }) s := &e.state s.wg.Wait() s.wWg.Wait() @@ -422,10 +418,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } - e.init.Do(func() { - e.o.setDefault() - e.initialize() - }) + e.init.Do(e.initialize) enc := <-e.encoders defer func() { // Release encoder reference to last block. diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 40eb45733..0ff970dac 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -39,9 +39,11 @@ func (o *encoderOptions) setDefault() { func (o encoderOptions) encoder() encoder { switch o.level { case SpeedDefault: - return &doubleFastEncoder{fastEncoder: fastEncoder{maxMatchOff: int32(o.windowSize)}} + return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}} + case SpeedBetterCompression: + return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} case SpeedFastest: - return &fastEncoder{maxMatchOff: int32(o.windowSize)} + return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} } panic("unknown compression level") } @@ -67,7 +69,7 @@ func WithEncoderConcurrency(n int) EOption { } // WithWindowSize will set the maximum allowed back-reference distance. -// The value must be a power of two between WindowSizeMin and WindowSizeMax. +// The value must be a power of two between MinWindowSize and MaxWindowSize. // A larger value will enable better compression but allocate more memory and, // for above-default values, take considerably longer. // The default value is determined by the compression level. @@ -130,18 +132,18 @@ const ( // This is roughly equivalent to the default Zstandard mode (level 3). SpeedDefault + // SpeedBetterCompression will yield better compression than the default. + // Currently it is about zstd level 7-8 with ~ 2x-3x the default CPU usage. + // By using this, notice that CPU usage may go up in the future. + SpeedBetterCompression + // speedLast should be kept as the last actual compression option. // The is not for external usage, but is used to keep track of the valid options. speedLast - // SpeedBetterCompression will (in the future) yield better compression than the default, - // but at approximately 4x the CPU usage of the default. - // For now this is not implemented. - SpeedBetterCompression = SpeedDefault - // SpeedBestCompression will choose the best available compression option. // For now this is not implemented. - SpeedBestCompression = SpeedDefault + SpeedBestCompression = SpeedBetterCompression ) // EncoderLevelFromString will convert a string representation of an encoding level back @@ -163,8 +165,10 @@ func EncoderLevelFromZstd(level int) EncoderLevel { switch { case level < 3: return SpeedFastest - case level >= 3: + case level >= 3 && level < 6: return SpeedDefault + case level > 5: + return SpeedBetterCompression } return SpeedDefault } @@ -176,6 +180,8 @@ func (e EncoderLevel) String() string { return "fastest" case SpeedDefault: return "default" + case SpeedBetterCompression: + return "better" default: return "invalid" } diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 5e0b64ccc..0807719c8 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -87,6 +87,17 @@ func printf(format string, a ...interface{}) { } } +// matchLenFast does matching, but will not match the last up to 7 bytes. +func matchLenFast(a, b []byte) int { + endI := len(a) & (math.MaxInt32 - 7) + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + return i + bits.TrailingZeros64(diff)>>3 + } + } + return endI +} + // matchLen returns the maximum length. // a must be the shortest of the two. // The function also returns whether all bytes matched. @@ -97,33 +108,18 @@ func matchLen(a, b []byte) int { return i + (bits.TrailingZeros64(diff) >> 3) } } + checked := (len(a) >> 3) << 3 a = a[checked:] b = b[checked:] - // TODO: We could do a 4 check. for i := range a { if a[i] != b[i] { - return int(i) + checked + return i + checked } } return len(a) + checked } -// matchLen returns a match length in src between index s and t -func matchLenIn(src []byte, s, t int32) int32 { - s1 := len(src) - b := src[t:] - a := src[s:s1] - b = b[:len(a)] - // Extend the match to be as long as possible. - for i := range a { - if a[i] != b[i] { - return int32(i) - } - } - return int32(len(a)) -} - func load3232(b []byte, i int32) uint32 { // Help the compiler eliminate bounds checks on the read so it can be done in a single read. b = b[i:] diff --git a/vendor/github.com/klauspost/pgzip/README.md b/vendor/github.com/klauspost/pgzip/README.md index 81000996c..171b978fd 100644 --- a/vendor/github.com/klauspost/pgzip/README.md +++ b/vendor/github.com/klauspost/pgzip/README.md @@ -39,7 +39,6 @@ You might need to get/update the dependencies: ``` go get -u github.com/klauspost/compress -go get -u github.com/klauspost/crc32 ``` Usage @@ -65,7 +64,7 @@ Changes in [github.com/klauspost/compress](https://github.com/klauspost/compress ## Compression The simplest way to use this is to simply do the same as you would when using [compress/gzip](http://golang.org/pkg/compress/gzip). -To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(250000, 16), meaning blocks are split at 250000 bytes and up to 16 blocks can be processing at once before the writer blocks. +To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(1MB, runtime.GOMAXPROCS(0)), meaning blocks are split at 1 MB and up to the number of CPU threads blocks can be processing at once before the writer blocks. Example: @@ -99,19 +98,19 @@ See my blog post in [Benchmarks of Golang Gzip](https://blog.klauspost.com/go-gz Compression cost is usually about 0.2% with default settings with a block size of 250k. -Example with GOMAXPROC set to 8 (quad core with 8 hyperthreads) +Example with GOMAXPROC set to 32 (16 core CPU) Content is [Matt Mahoneys 10GB corpus](http://mattmahoney.net/dc/10gb.html). Compression level 6. Compressor | MB/sec | speedup | size | size overhead (lower=better) ------------|----------|---------|------|--------- -[gzip](http://golang.org/pkg/compress/gzip) (golang) | 7.21MB/s | 1.0x | 4786608902 | 0% -[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 10.98MB/s | 1.52x | 4781331645 | -0.11% -[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 50.76MB/s|7.04x | 4784121440 | -0.052% -[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 38.65MB/s | 5.36x | 4924899484 | 2.889% -[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 32.00MB/s | 4.44x | 4791226567 | 0.096% +[gzip](http://golang.org/pkg/compress/gzip) (golang) | 15.44MB/s (1 thread) | 1.0x | 4781329307 | 0% +[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 135.04MB/s (1 thread) | 8.74x | 4894858258 | +2.37% +[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 1573.23MB/s| 101.9x | 4902285651 | +2.53% +[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 361.40MB/s | 23.4x | 4869686090 | +1.85% +[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 306.01MB/s | 19.8x | 4786890417 | +0.12% -pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression) mode, that will allow compression at ~150MB per core per second, independent of the content. +pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression-huffman-only) mode, that will allow compression at ~250MB per core per second, independent of the content. See the [complete sheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) for different content types and compression settings. diff --git a/vendor/github.com/klauspost/pgzip/gzip.go b/vendor/github.com/klauspost/pgzip/gzip.go index 85d14e9cb..bb2e33941 100644 --- a/vendor/github.com/klauspost/pgzip/gzip.go +++ b/vendor/github.com/klauspost/pgzip/gzip.go @@ -11,6 +11,7 @@ import ( "hash" "hash/crc32" "io" + "runtime" "sync" "time" @@ -18,9 +19,9 @@ import ( ) const ( - defaultBlockSize = 256 << 10 + defaultBlockSize = 1 << 20 tailSize = 16384 - defaultBlocks = 16 + defaultBlocks = 4 ) // These constants are copied from the flate package, so that code that imports @@ -68,8 +69,8 @@ type result struct { // With this you can control the approximate size of your blocks, // as well as how many you want to be processing in parallel. // -// Default values for this is SetConcurrency(250000, 16), -// meaning blocks are split at 250000 bytes and up to 16 blocks +// Default values for this is SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)), +// meaning blocks are split at 1 MB and up to the number of CPU threads // can be processing at once before the writer blocks. func (z *Writer) SetConcurrency(blockSize, blocks int) error { if blockSize <= tailSize { @@ -115,7 +116,7 @@ func NewWriterLevel(w io.Writer, level int) (*Writer, error) { return nil, fmt.Errorf("gzip: invalid compression level: %d", level) } z := new(Writer) - z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)) z.init(w, level) return z, nil } @@ -174,7 +175,7 @@ func (z *Writer) Reset(w io.Writer) { if z.results != nil && !z.closed { close(z.results) } - z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.SetConcurrency(defaultBlockSize, runtime.GOMAXPROCS(0)) z.init(w, z.level) } @@ -239,36 +240,36 @@ func (z *Writer) writeString(s string) (err error) { // compressCurrent will compress the data currently buffered // This should only be called from the main writer/flush/closer func (z *Writer) compressCurrent(flush bool) { + c := z.currentBuffer + if len(c) > z.blockSize { + // This can never happen through the public interface. + panic("len(z.currentBuffer) > z.blockSize (most likely due to concurrent Write race)") + } + r := result{} r.result = make(chan []byte, 1) r.notifyWritten = make(chan struct{}, 0) + // Reserve a result slot select { case z.results <- r: case <-z.pushedErr: return } - // If block given is more than twice the block size, split it. - c := z.currentBuffer - if len(c) > z.blockSize*2 { - c = c[:z.blockSize] - z.wg.Add(1) - go z.compressBlock(c, z.prevTail, r, false) - z.prevTail = c[len(c)-tailSize:] - z.currentBuffer = z.currentBuffer[z.blockSize:] - z.compressCurrent(flush) - // Last one flushes if needed - return - } - z.wg.Add(1) - go z.compressBlock(c, z.prevTail, r, z.closed) + tail := z.prevTail if len(c) > tailSize { - z.prevTail = c[len(c)-tailSize:] + buf := z.dstPool.Get().([]byte) // Put in .compressBlock + // Copy tail from current buffer before handing the buffer over to the + // compressBlock goroutine. + buf = append(buf[:0], c[len(c)-tailSize:]...) + z.prevTail = buf } else { z.prevTail = nil } - z.currentBuffer = z.dstPool.Get().([]byte) + go z.compressBlock(c, tail, r, z.closed) + + z.currentBuffer = z.dstPool.Get().([]byte) // Put in .compressBlock z.currentBuffer = z.currentBuffer[:0] // Wait if flushing @@ -358,29 +359,37 @@ func (z *Writer) Write(p []byte) (int, error) { // Start receiving data from compressors go func() { listen := z.results + var failed bool for { r, ok := <-listen // If closed, we are finished. if !ok { return } + if failed { + close(r.notifyWritten) + continue + } buf := <-r.result n, err := z.w.Write(buf) if err != nil { z.pushError(err) close(r.notifyWritten) - return + failed = true + continue } if n != len(buf) { z.pushError(fmt.Errorf("gzip: short write %d should be %d", n, len(buf))) + failed = true close(r.notifyWritten) - return + continue } z.dstPool.Put(buf) close(r.notifyWritten) } }() - z.currentBuffer = make([]byte, 0, z.blockSize) + z.currentBuffer = z.dstPool.Get().([]byte) + z.currentBuffer = z.currentBuffer[:0] } q := p for len(q) > 0 { @@ -390,7 +399,10 @@ func (z *Writer) Write(p []byte) (int, error) { } z.digest.Write(q[:length]) z.currentBuffer = append(z.currentBuffer, q[:length]...) - if len(z.currentBuffer) >= z.blockSize { + if len(z.currentBuffer) > z.blockSize { + panic("z.currentBuffer too large (most likely due to concurrent Write race)") + } + if len(z.currentBuffer) == z.blockSize { z.compressCurrent(false) if err := z.checkError(); err != nil { return len(p) - len(q) - length, err @@ -410,12 +422,13 @@ func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) { close(r.result) z.wg.Done() }() - buf := z.dstPool.Get().([]byte) + buf := z.dstPool.Get().([]byte) // Corresponding Put in .Write's result writer dest := bytes.NewBuffer(buf[:0]) - compressor := z.dictFlatePool.Get().(*flate.Writer) + compressor := z.dictFlatePool.Get().(*flate.Writer) // Put below compressor.ResetDict(dest, prevTail) compressor.Write(p) + z.dstPool.Put(p) // Corresponding Get in .Write and .compressCurrent err := compressor.Flush() if err != nil { @@ -429,7 +442,12 @@ func (z *Writer) compressBlock(p, prevTail []byte, r result, closed bool) { return } } - z.dictFlatePool.Put(compressor) + z.dictFlatePool.Put(compressor) // Get above + + if prevTail != nil { + z.dstPool.Put(prevTail) // Get in .compressCurrent + } + // Read back buffer buf = dest.Bytes() r.result <- buf diff --git a/vendor/github.com/openshift/imagebuilder/builder.go b/vendor/github.com/openshift/imagebuilder/builder.go index 5a2d0d539..81d7b8421 100644 --- a/vendor/github.com/openshift/imagebuilder/builder.go +++ b/vendor/github.com/openshift/imagebuilder/builder.go @@ -288,8 +288,12 @@ func NewBuilder(args map[string]string) *Builder { for k, v := range builtinAllowedBuildArgs { allowed[k] = v } + provided := make(map[string]string) + for k, v := range args { + provided[k] = v + } return &Builder{ - Args: args, + Args: provided, AllowedArgs: allowed, } } diff --git a/vendor/github.com/openshift/imagebuilder/dispatchers.go b/vendor/github.com/openshift/imagebuilder/dispatchers.go index ff365848a..e7f2f97bf 100644 --- a/vendor/github.com/openshift/imagebuilder/dispatchers.go +++ b/vendor/github.com/openshift/imagebuilder/dispatchers.go @@ -19,6 +19,7 @@ import ( docker "github.com/fsouza/go-dockerclient" + "github.com/containerd/containerd/platforms" "github.com/openshift/imagebuilder/signal" "github.com/openshift/imagebuilder/strslice" ) @@ -27,6 +28,27 @@ var ( obRgex = regexp.MustCompile(`(?i)^\s*ONBUILD\s*`) ) +var localspec = platforms.DefaultSpec() + +// https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope +var builtinBuildArgs = map[string]string{ + "TARGETPLATFORM": localspec.OS + "/" + localspec.Architecture, + "TARGETOS": localspec.OS, + "TARGETARCH": localspec.Architecture, + "TARGETVARIANT": localspec.Variant, + "BUILDPLATFORM": localspec.OS + "/" + localspec.Architecture, + "BUILDOS": localspec.OS, + "BUILDARCH": localspec.Architecture, + "BUILDVARIANT": localspec.Variant, +} + +func init() { + if localspec.Variant != "" { + builtinBuildArgs["TARGETPLATFORM"] = builtinBuildArgs["TARGETPLATFORM"] + "/" + localspec.Variant + builtinBuildArgs["BUILDPLATFORM"] = builtinBuildArgs["BUILDPLATFORM"] + "/" + localspec.Variant + } +} + // ENV foo bar // // Sets the environment variable foo to bar, also makes interpolation @@ -131,14 +153,16 @@ func add(b *Builder, args []string, attributes map[string]bool, flagArgs []strin var chown string last := len(args) - 1 dest := makeAbsolute(args[last], b.RunConfig.WorkingDir) - if len(flagArgs) > 0 { - for _, arg := range flagArgs { - switch { - case strings.HasPrefix(arg, "--chown="): - chown = strings.TrimPrefix(arg, "--chown=") - default: - return fmt.Errorf("ADD only supports the --chown=<uid:gid> flag") - } + for _, a := range flagArgs { + arg, err := ProcessWord(a, b.Env) + if err != nil { + return err + } + switch { + case strings.HasPrefix(arg, "--chown="): + chown = strings.TrimPrefix(arg, "--chown=") + default: + return fmt.Errorf("ADD only supports the --chown=<uid:gid> flag") } } b.PendingCopies = append(b.PendingCopies, Copy{Src: args[0:last], Dest: dest, Download: true, Chown: chown}) @@ -157,16 +181,18 @@ func dispatchCopy(b *Builder, args []string, attributes map[string]bool, flagArg dest := makeAbsolute(args[last], b.RunConfig.WorkingDir) var chown string var from string - if len(flagArgs) > 0 { - for _, arg := range flagArgs { - switch { - case strings.HasPrefix(arg, "--chown="): - chown = strings.TrimPrefix(arg, "--chown=") - case strings.HasPrefix(arg, "--from="): - from = strings.TrimPrefix(arg, "--from=") - default: - return fmt.Errorf("COPY only supports the --chown=<uid:gid> and the --from=<image|stage> flags") - } + for _, a := range flagArgs { + arg, err := ProcessWord(a, b.Env) + if err != nil { + return err + } + switch { + case strings.HasPrefix(arg, "--chown="): + chown = strings.TrimPrefix(arg, "--chown=") + case strings.HasPrefix(arg, "--from="): + from = strings.TrimPrefix(arg, "--from=") + default: + return fmt.Errorf("COPY only supports the --chown=<uid:gid> and the --from=<image|stage> flags") } } b.PendingCopies = append(b.PendingCopies, Copy{From: from, Src: args[0:last], Dest: dest, Download: false, Chown: chown}) @@ -516,6 +542,8 @@ func healthcheck(b *Builder, args []string, attributes map[string]bool, flagArgs return nil } +var targetArgs = []string{"TARGETOS", "TARGETARCH", "TARGETVARIANT"} + // ARG name[=value] // // Adds the variable foo to the trusted list of variables that can be passed @@ -543,6 +571,26 @@ func arg(b *Builder, args []string, attributes map[string]bool, flagArgs []strin name = parts[0] value = parts[1] hasDefault = true + if name == "TARGETPLATFORM" { + p, err := platforms.Parse(value) + if err != nil { + return fmt.Errorf("error parsing TARGETPLATFORM argument") + } + for _, val := range targetArgs { + b.AllowedArgs[val] = true + } + b.Args["TARGETPLATFORM"] = p.OS + "/" + p.Architecture + b.Args["TARGETOS"] = p.OS + b.Args["TARGETARCH"] = p.Architecture + b.Args["TARGETVARIANT"] = p.Variant + if p.Variant != "" { + b.Args["TARGETPLATFORM"] = b.Args["TARGETPLATFORM"] + "/" + p.Variant + } + } + } else if val, ok := builtinBuildArgs[arg]; ok { + name = arg + value = val + hasDefault = true } else { name = arg hasDefault = false diff --git a/vendor/github.com/openshift/imagebuilder/vendor.conf b/vendor/github.com/openshift/imagebuilder/vendor.conf index c3f7d1a6b..8074ce80a 100644 --- a/vendor/github.com/openshift/imagebuilder/vendor.conf +++ b/vendor/github.com/openshift/imagebuilder/vendor.conf @@ -1,4 +1,5 @@ github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 +github.com/containerd/containerd v1.3.0 github.com/containers/storage v1.2 github.com/docker/docker b68221c37ee597950364788204546f9c9d0e46a1 github.com/docker/go-connections 97c2040d34dfae1d1b1275fa3a78dbdd2f41cf7e @@ -18,3 +19,7 @@ golang.org/x/crypto ff983b9c42bc9fbf91556e191cc8efb585c16908 golang.org/x/net 45ffb0cd1ba084b73e26dee67e667e1be5acce83 golang.org/x/sys 7fbe1cd0fcc20051e1fcb87fbabec4a1bacaaeba k8s.io/klog 8e90cee79f823779174776412c13478955131846 +google.golang.org/grpc 6eaf6f47437a6b4e2153a190160ef39a92c7eceb # v1.23.0 +github.com/golang/protobuf v1.2.0 +google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 + diff --git a/vendor/modules.txt b/vendor/modules.txt index ad8ebb4e4..b143eea5a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -44,6 +44,8 @@ github.com/checkpoint-restore/go-criu/rpc github.com/containerd/cgroups/stats/v1 # github.com/containerd/containerd v1.3.0 github.com/containerd/containerd/errdefs +github.com/containerd/containerd/log +github.com/containerd/containerd/platforms # github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc github.com/containerd/continuity/fs github.com/containerd/continuity/syscallx @@ -62,22 +64,25 @@ github.com/containernetworking/plugins/pkg/ns github.com/containernetworking/plugins/pkg/utils/hwaddr github.com/containernetworking/plugins/plugins/ipam/host-local/backend github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator -# github.com/containers/buildah v1.14.2 +# github.com/containers/buildah v1.14.3 github.com/containers/buildah github.com/containers/buildah/bind github.com/containers/buildah/chroot github.com/containers/buildah/docker github.com/containers/buildah/imagebuildah +github.com/containers/buildah/manifests github.com/containers/buildah/pkg/blobcache github.com/containers/buildah/pkg/chrootuser github.com/containers/buildah/pkg/cli github.com/containers/buildah/pkg/formats +github.com/containers/buildah/pkg/manifests github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/parse github.com/containers/buildah/pkg/secrets +github.com/containers/buildah/pkg/supplemented github.com/containers/buildah/pkg/umask github.com/containers/buildah/util -# github.com/containers/common v0.4.2 +# github.com/containers/common v0.6.1 github.com/containers/common/pkg/capabilities github.com/containers/common/pkg/config github.com/containers/common/pkg/unshare @@ -142,7 +147,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.16.3 +# github.com/containers/storage v1.16.5 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -262,7 +267,7 @@ github.com/docker/spdystream github.com/docker/spdystream/spdy # github.com/etcd-io/bbolt v1.3.3 github.com/etcd-io/bbolt -# github.com/fsnotify/fsnotify v1.4.7 +# github.com/fsnotify/fsnotify v1.4.9 github.com/fsnotify/fsnotify # github.com/fsouza/go-dockerclient v1.6.3 github.com/fsouza/go-dockerclient @@ -313,14 +318,14 @@ github.com/inconshreveable/mousetrap github.com/ishidawataru/sctp # github.com/json-iterator/go v1.1.9 github.com/json-iterator/go -# github.com/klauspost/compress v1.10.2 +# github.com/klauspost/compress v1.10.3 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 github.com/klauspost/compress/snappy github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/klauspost/pgzip v1.2.1 +# github.com/klauspost/pgzip v1.2.2 github.com/klauspost/pgzip # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences @@ -410,7 +415,7 @@ github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk # github.com/openshift/api v0.0.0-20200106203948-7ab22a2c8316 github.com/openshift/api/config/v1 -# github.com/openshift/imagebuilder v1.1.1 +# github.com/openshift/imagebuilder v1.1.2 github.com/openshift/imagebuilder github.com/openshift/imagebuilder/dockerfile/command github.com/openshift/imagebuilder/dockerfile/parser @@ -631,9 +636,9 @@ gopkg.in/square/go-jose.v2/json gopkg.in/tomb.v1 # gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v2 -# k8s.io/api v0.17.3 +# k8s.io/api v0.17.4 k8s.io/api/core/v1 -# k8s.io/apimachinery v0.17.3 +# k8s.io/apimachinery v0.17.4 k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/apis/meta/v1 |