From bd61c779caa6eed613d7e8595f0a6892bc8fa306 Mon Sep 17 00:00:00 2001 From: Urvashi Mohnani Date: Tue, 20 Nov 2018 20:07:17 +0000 Subject: Vendor in latest containers/buildah Pulls in fix for COPY --from when using --layers Signed-off-by: Urvashi Mohnani --- vendor.conf | 2 +- vendor/github.com/containers/buildah/chroot/run.go | 43 ++++++++++++++++------ .../containers/buildah/imagebuildah/build.go | 32 ++++++++++++++-- .../buildah/imagebuildah/chroot_symlink.go | 7 ++++ .../containers/buildah/imagebuildah/util.go | 25 +++++++++++++ vendor/github.com/containers/buildah/run.go | 15 -------- vendor/github.com/containers/buildah/vendor.conf | 8 ++-- .../github.com/openshift/imagebuilder/builder.go | 39 +++++++++++++++++++- .../openshift/imagebuilder/dispatchers.go | 23 +++++++++--- .../github.com/openshift/imagebuilder/evaluator.go | 3 +- 10 files changed, 153 insertions(+), 44 deletions(-) diff --git a/vendor.conf b/vendor.conf index c8e968648..0c05e792c 100644 --- a/vendor.conf +++ b/vendor.conf @@ -92,7 +92,7 @@ k8s.io/kube-openapi 275e2ce91dec4c05a4094a7b1daee5560b555ac9 https://github.com/ k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e https://github.com/kubernetes/utils github.com/mrunalp/fileutils master github.com/varlink/go master -github.com/containers/buildah 795d43e60e5a1ab283981b79eeda1dd14a65a0bd +github.com/containers/buildah 2ac987a52ff8412fb8f2908a191009751a6a1c62 github.com/Nvveen/Gotty master github.com/fsouza/go-dockerclient master github.com/openshift/imagebuilder master diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index 8cfefb3de..6a1400e61 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -955,6 +955,20 @@ func setRlimits(spec *specs.Spec, onlyLower, onlyRaise bool) error { return nil } +func makeReadOnly(mntpoint string, flags uintptr) error { + var fs unix.Statfs_t + // Make sure it's read-only. + if err := unix.Statfs(mntpoint, &fs); err != nil { + return errors.Wrapf(err, "error checking if directory %q was bound read-only", mntpoint) + } + if fs.Flags&unix.ST_RDONLY == 0 { + if err := unix.Mount(mntpoint, mntpoint, "bind", flags|unix.MS_REMOUNT, ""); err != nil { + return errors.Wrapf(err, "error remounting %s in mount namespace read-only", mntpoint) + } + } + return nil +} + // setupChrootBindMounts actually bind mounts things under the rootfs, and returns a // callback that will clean up its work. func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func() error, err error) { @@ -976,7 +990,7 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( bindFlags := commonFlags | unix.MS_NODEV devFlags := commonFlags | unix.MS_NOEXEC | unix.MS_NOSUID | unix.MS_RDONLY procFlags := devFlags | unix.MS_NODEV - sysFlags := devFlags | unix.MS_NODEV | unix.MS_RDONLY + sysFlags := devFlags | unix.MS_NODEV // Bind /dev read-only. subDev := filepath.Join(spec.Root.Path, "/dev") @@ -1030,13 +1044,22 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( return undoBinds, errors.Wrapf(err, "error bind mounting /sys from host into mount namespace") } } - // Make sure it's read-only. - if err = unix.Statfs(subSys, &fs); err != nil { - return undoBinds, errors.Wrapf(err, "error checking if directory %q was bound read-only", subSys) + if err := makeReadOnly(subSys, sysFlags); err != nil { + return undoBinds, err } - if fs.Flags&unix.ST_RDONLY == 0 { - if err := unix.Mount(subSys, subSys, "bind", sysFlags|unix.MS_REMOUNT, ""); err != nil { - return undoBinds, errors.Wrapf(err, "error remounting /sys in mount namespace read-only") + + mnts, _ := mount.GetMounts() + for _, m := range mnts { + if !strings.HasPrefix(m.Mountpoint, "/sys/") && + m.Mountpoint != "/sys" { + continue + } + subSys := filepath.Join(spec.Root.Path, m.Mountpoint) + if err := unix.Mount(m.Mountpoint, subSys, "bind", sysFlags, ""); err != nil { + return undoBinds, errors.Wrapf(err, "error bind mounting /sys from host into mount namespace") + } + if err := makeReadOnly(subSys, sysFlags); err != nil { + return undoBinds, err } } logrus.Debugf("bind mounted %q to %q", "/sys", filepath.Join(spec.Root.Path, "/sys")) @@ -1044,10 +1067,6 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( // Add /sys/fs/selinux to the set of masked paths, to ensure that we don't have processes // attempting to interact with labeling, when they aren't allowed to do so. spec.Linux.MaskedPaths = append(spec.Linux.MaskedPaths, "/sys/fs/selinux") - // Add /sys/fs/cgroup to the set of masked paths, to ensure that we don't have processes - // attempting to mess with cgroup configuration, when they aren't allowed to do so. - spec.Linux.MaskedPaths = append(spec.Linux.MaskedPaths, "/sys/fs/cgroup") - // Bind mount in everything we've been asked to mount. for _, m := range spec.Mounts { // Skip anything that we just mounted. @@ -1143,7 +1162,7 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( logrus.Debugf("mounted a tmpfs to %q", target) } if err = unix.Statfs(target, &fs); err != nil { - return undoBinds, errors.Wrapf(err, "error checking if directory %q was bound read-only", subSys) + return undoBinds, errors.Wrapf(err, "error checking if directory %q was bound read-only", target) } if uintptr(fs.Flags)&expectedFlags != expectedFlags { if err := unix.Mount(target, target, "bind", requestFlags|unix.MS_REMOUNT, ""); err != nil { diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index 292ff9541..701241683 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -222,7 +222,7 @@ type Executor struct { forceRmIntermediateCtrs bool containerIDs []string // Stores the IDs of the successful intermediate containers used during layer build imageMap map[string]string // Used to map images that we create to handle the AS construct. - + copyFrom string // Used to keep track of the --from flag from COPY and ADD } // withName creates a new child executor that will be used whenever a COPY statement uses --from=NAME. @@ -826,6 +826,18 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error err error imgID string ) + + b.copyFrom = "" + // Check if --from exists in the step command of COPY or ADD + // If it exists, set b.copyfrom to that value + for _, n := range step.Flags { + if strings.Contains(n, "--from") && (step.Command == "copy" || step.Command == "add") { + arr := strings.Split(n, "=") + b.copyFrom = b.named[arr[1]].mountPoint + break + } + } + // checkForLayers will be true if b.layers is true and a cached intermediate image is found. // checkForLayers is set to false when either there is no cached image or a break occurs where // the instructions in the Dockerfile change from a previous build. @@ -848,6 +860,7 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error if err := b.copyExistingImage(ctx, cacheID); err != nil { return err } + b.containerIDs = append(b.containerIDs, b.builder.ContainerID) break } @@ -1009,6 +1022,11 @@ func (b *Executor) getFilesToCopy(node *parser.Node) ([]string, error) { currNode = currNode.Next continue } + if b.copyFrom != "" { + src = append(src, filepath.Join(b.copyFrom, currNode.Value)) + currNode = currNode.Next + continue + } matches, err := filepath.Glob(filepath.Join(b.contextDir, currNode.Value)) if err != nil { return nil, errors.Wrapf(err, "error finding match for pattern %q", currNode.Value) @@ -1049,7 +1067,12 @@ func (b *Executor) copiedFilesMatch(node *parser.Node, historyTime *time.Time) ( // Change the time format to ensure we don't run into a parsing error when converting again from string // to time.Time. It is a known Go issue that the conversions cause errors sometimes, so specifying a particular // time format here when converting to a string. - timeIsGreater, err := resolveModifiedTime(b.contextDir, item, historyTime.Format(time.RFC3339Nano)) + // If the COPY has --from in the command, change the rootdir to mountpoint of the container it is copying from + rootdir := b.contextDir + if b.copyFrom != "" { + rootdir = b.copyFrom + } + timeIsGreater, err := resolveModifiedTime(rootdir, item, historyTime.Format(time.RFC3339Nano)) if err != nil { return false, errors.Wrapf(err, "error resolving symlinks and comparing modified times: %q", item) } @@ -1342,7 +1365,10 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options BuildOpt return "", nil, errors.Wrapf(err, "error creating build executor") } b := imagebuilder.NewBuilder(options.Args) - stages := imagebuilder.NewStages(mainNode, b) + stages, err := imagebuilder.NewStages(mainNode, b) + if err != nil { + return "", nil, errors.Wrap(err, "error reading multiple stages") + } return exec.Build(ctx, stages) } diff --git a/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go b/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go index 2269b8dcc..edb5837db 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go +++ b/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go @@ -140,6 +140,13 @@ func modTimeIsGreater(rootdir, path string, historyTime string) (bool, error) { // Since we are chroot in rootdir, only want the path of the actual filename, i.e path - rootdir. // +1 to account for the extra "/" (e.g rootdir=/home/user/mydir, path=/home/user/mydir/myfile.json) err = filepath.Walk(path[len(rootdir)+1:], func(path string, info os.FileInfo, err error) error { + // If using cached images, it is possible for files that are being copied to come from + // previous build stages. But if using cached images, then the copied file won't exist + // since a container won't have been created for the previous build stage and info will be nil. + // In that case just return nil and continue on with using the cached image for the whole build process. + if info == nil { + return nil + } modTime := info.ModTime() if info.Mode()&os.ModeSymlink == os.ModeSymlink { // Evaluate any symlink that occurs to get updated modified information diff --git a/vendor/github.com/containers/buildah/imagebuildah/util.go b/vendor/github.com/containers/buildah/imagebuildah/util.go index 35dc5438a..4f5301b73 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/util.go +++ b/vendor/github.com/containers/buildah/imagebuildah/util.go @@ -111,3 +111,28 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err func InitReexec() bool { return buildah.InitReexec() } + +// ReposToMap parses the specified repotags and returns a map with repositories +// as keys and the corresponding arrays of tags as values. +func ReposToMap(repotags []string) map[string][]string { + // map format is repo -> tag + repos := make(map[string][]string) + for _, repo := range repotags { + var repository, tag string + if strings.Contains(repo, ":") { + li := strings.LastIndex(repo, ":") + repository = repo[0:li] + tag = repo[li+1:] + } else if len(repo) > 0 { + repository = repo + tag = "" + } else { + logrus.Warnf("Found image with empty name") + } + repos[repository] = append(repos[repository], tag) + } + if len(repos) == 0 { + repos[""] = []string{""} + } + return repos +} diff --git a/vendor/github.com/containers/buildah/run.go b/vendor/github.com/containers/buildah/run.go index 636a204b3..5d2cd6a32 100644 --- a/vendor/github.com/containers/buildah/run.go +++ b/vendor/github.com/containers/buildah/run.go @@ -1104,14 +1104,6 @@ func (b *Builder) Run(command []string, options RunOptions) error { switch isolation { case IsolationOCI: - // The default is --rootless=auto, which makes troubleshooting a bit harder. - // rootlessFlag := []string{"--rootless=false"} - // for _, arg := range options.Args { - // if strings.HasPrefix(arg, "--rootless") { - // rootlessFlag = nil - // } - // } - // options.Args = append(options.Args, rootlessFlag...) var moreCreateArgs []string if options.NoPivot { moreCreateArgs = []string{"--no-pivot"} @@ -1125,13 +1117,6 @@ func (b *Builder) Run(command []string, options RunOptions) error { if err := setupRootlessSpecChanges(spec, path, rootUID, rootGID); err != nil { return err } - rootlessFlag := []string{"--rootless=true"} - for _, arg := range options.Args { - if strings.HasPrefix(arg, "--rootless") { - rootlessFlag = nil - } - } - options.Args = append(options.Args, rootlessFlag...) err = b.runUsingRuntimeSubproc(isolation, options, configureNetwork, configureNetworks, []string{"--no-new-keyring"}, spec, mountPoint, path, Package+"-"+filepath.Base(path)) default: err = errors.Errorf("don't know how to run this command") diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index c4410d0af..185cde449 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,9 +3,9 @@ github.com/blang/semver master github.com/BurntSushi/toml master github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 -github.com/containers/image 5e5b67d6b1cf43cc349128ec3ed7d5283a6cc0d1 -github.com/containers/libpod e75469ab99c48e9fbe2b36ade229d384bdea9144 -github.com/containers/storage 09abf3a26b8a3aa69e29fd7faeb260b98d675759 +github.com/containers/image de7be82ee3c7fb676bf6cfdc9090be7cc28f404c +github.com/containers/libpod fe4f09493f41f675d24c969d1b60d1a6a45ddb9e +github.com/containers/storage 3161726d1db0d0d4e86a9667dd476f09b997f497 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 @@ -38,7 +38,7 @@ github.com/opencontainers/runtime-spec v1.0.0 github.com/opencontainers/runtime-tools master github.com/opencontainers/selinux master github.com/openshift/imagebuilder master -github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 +github.com/ostreedev/ostree-go 9ab99253d365aac3a330d1f7281cf29f3d22820b github.com/pborman/uuid master github.com/pkg/errors master github.com/pquerna/ffjson d49c2bc1aa135aad0c6f4fc2056623ec78f5d5ac diff --git a/vendor/github.com/openshift/imagebuilder/builder.go b/vendor/github.com/openshift/imagebuilder/builder.go index 1c1afb119..d37965df6 100644 --- a/vendor/github.com/openshift/imagebuilder/builder.go +++ b/vendor/github.com/openshift/imagebuilder/builder.go @@ -172,8 +172,11 @@ type Stage struct { Node *parser.Node } -func NewStages(node *parser.Node, b *Builder) Stages { +func NewStages(node *parser.Node, b *Builder) (Stages, error) { var stages Stages + if err := b.extractHeadingArgsFromNode(node); err != nil { + return stages, err + } for i, root := range SplitBy(node, command.From) { name, _ := extractNameFromNode(root.Children[0]) if len(name) == 0 { @@ -189,7 +192,36 @@ func NewStages(node *parser.Node, b *Builder) Stages { Node: root, }) } - return stages + return stages, nil +} + +func (b *Builder) extractHeadingArgsFromNode(node *parser.Node) error { + var args []*parser.Node + var children []*parser.Node + extract := true + for _, child := range node.Children { + if extract && child.Value == command.Arg { + args = append(args, child) + } else { + if child.Value == command.From { + extract = false + } + children = append(children, child) + } + } + + for _, c := range args { + step := b.Step() + if err := step.Resolve(c); err != nil { + return err + } + if err := b.Run(step, NoopExecutor, false); err != nil { + return err + } + } + + node.Children = children + return nil } func extractNameFromNode(node *parser.Node) (string, bool) { @@ -345,6 +377,9 @@ var ErrNoFROM = fmt.Errorf("no FROM statement found") // is set to the first From found, or left unchanged if already // set. func (b *Builder) From(node *parser.Node) (string, error) { + if err := b.extractHeadingArgsFromNode(node); err != nil { + return "", err + } children := SplitChildren(node, command.From) switch { case len(children) == 0: diff --git a/vendor/github.com/openshift/imagebuilder/dispatchers.go b/vendor/github.com/openshift/imagebuilder/dispatchers.go index 068d5cc6f..f6510c2fd 100644 --- a/vendor/github.com/openshift/imagebuilder/dispatchers.go +++ b/vendor/github.com/openshift/imagebuilder/dispatchers.go @@ -27,11 +27,6 @@ var ( obRgex = regexp.MustCompile(`(?i)^\s*ONBUILD\s*`) ) -// dispatch with no layer / parsing. This is effectively not a command. -func nullDispatch(b *Builder, args []string, attributes map[string]bool, flagArgs []string, original string) error { - return nil -} - // ENV foo bar // // Sets the environment variable foo to bar, also makes interpolation @@ -181,6 +176,17 @@ func from(b *Builder, args []string, attributes map[string]bool, flagArgs []stri } name := args[0] + + // Support ARG before from + argStrs := []string{} + for n, v := range b.Args { + argStrs = append(argStrs, n+"="+v) + } + var err error + if name, err = ProcessWord(name, argStrs); err != nil { + return err + } + // Windows cannot support a container with no base image. if name == NoBaseImageSpecifier { if runtime.GOOS == "windows" { @@ -438,6 +444,7 @@ func healthcheck(b *Builder, args []string, attributes map[string]bool, flagArgs healthcheck := docker.HealthConfig{} flags := flag.NewFlagSet("", flag.ContinueOnError) + flags.String("start-period", "", "") flags.String("interval", "", "") flags.String("timeout", "", "") flRetries := flags.String("retries", "", "") @@ -462,6 +469,12 @@ func healthcheck(b *Builder, args []string, attributes map[string]bool, flagArgs return fmt.Errorf("Unknown type %#v in HEALTHCHECK (try CMD)", typ) } + period, err := parseOptInterval(flags.Lookup("start-period")) + if err != nil { + return err + } + healthcheck.StartPeriod = period + interval, err := parseOptInterval(flags.Lookup("interval")) if err != nil { return err diff --git a/vendor/github.com/openshift/imagebuilder/evaluator.go b/vendor/github.com/openshift/imagebuilder/evaluator.go index 83263127e..e1cd5d6d6 100644 --- a/vendor/github.com/openshift/imagebuilder/evaluator.go +++ b/vendor/github.com/openshift/imagebuilder/evaluator.go @@ -122,8 +122,7 @@ func (b *Step) Resolve(ast *parser.Node) error { envs := b.Env for ast.Next != nil { ast = ast.Next - var str string - str = ast.Value + str := ast.Value if replaceEnvAllowed[cmd] { var err error var words []string -- cgit v1.2.3-54-g00ecf