diff options
author | baude <bbaude@redhat.com> | 2018-04-03 12:34:19 -0500 |
---|---|---|
committer | Atomic Bot <atomic-devel@projectatomic.io> | 2018-04-10 13:31:59 +0000 |
commit | 1700f2b2381d9665810ed4764d0fe357150c5978 (patch) | |
tree | a7338bdc31da8ef48d337555911932b097c06f84 | |
parent | 998fd2ece0480e581e013124d0969a1af6305110 (diff) | |
download | podman-1700f2b2381d9665810ed4764d0fe357150c5978.tar.gz podman-1700f2b2381d9665810ed4764d0fe357150c5978.tar.bz2 podman-1700f2b2381d9665810ed4764d0fe357150c5978.zip |
Use buildah commit for podman commit
Resolves: #586 and #520
Signed-off-by: baude <bbaude@redhat.com>
Closes: #592
Approved by: mheon
-rw-r--r-- | cmd/podman/commit.go | 66 | ||||
-rw-r--r-- | docs/podman-commit.1.md | 16 | ||||
-rw-r--r-- | libpod/buildah/buildah.go | 232 | ||||
-rw-r--r-- | libpod/buildah/commit.go | 147 | ||||
-rw-r--r-- | libpod/buildah/common.go | 28 | ||||
-rw-r--r-- | libpod/buildah/config.go | 607 | ||||
-rw-r--r-- | libpod/buildah/image.go | 529 | ||||
-rw-r--r-- | libpod/buildah/util.go | 67 | ||||
-rw-r--r-- | libpod/container_api.go | 39 | ||||
-rw-r--r-- | libpod/container_commit.go | 103 | ||||
-rw-r--r-- | libpod/image/image.go | 8 |
11 files changed, 1754 insertions, 88 deletions
diff --git a/cmd/podman/commit.go b/cmd/podman/commit.go index 34a086004..57798156b 100644 --- a/cmd/podman/commit.go +++ b/cmd/podman/commit.go @@ -4,10 +4,13 @@ import ( "fmt" "io" "os" + "strings" - "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/projectatomic/libpod/libpod" + "github.com/projectatomic/libpod/libpod/buildah" "github.com/projectatomic/libpod/libpod/image" + "github.com/projectatomic/libpod/pkg/util" "github.com/urfave/cli" ) @@ -52,7 +55,6 @@ func commitCmd(c *cli.Context) error { if err := validateFlags(c, commitFlags); err != nil { return err } - runtime, err := getRuntime(c) if err != nil { return errors.Wrapf(err, "could not get runtime") @@ -60,52 +62,48 @@ func commitCmd(c *cli.Context) error { defer runtime.Shutdown(false) var ( - container string - reference string - writer io.Writer + writer io.Writer ) args := c.Args() - switch len(args) { - case 0: - return errors.Errorf("need to give container name or id") - case 1: - container = args[0] - case 2: - container = args[0] - reference = args[1] - default: - return errors.Errorf("too many arguments. Usage CONTAINER [REFERENCE]") + if len(args) != 2 { + return errors.Errorf("you must provide a container name or ID and a target image name") } - - changes := v1.ImageConfig{} + container := args[0] + reference := args[1] if c.IsSet("change") { - changes, err = getImageConfig(c.StringSlice("change")) - if err != nil { - return errors.Wrapf(err, "error adding config changes to image %q", container) + for _, change := range c.StringSlice("change") { + splitChange := strings.Split(strings.ToUpper(change), "=") + if !util.StringInSlice(splitChange[0], []string{"CMD", "ENTRYPOINT", "ENV", "EXPOSE", "LABEL", "STOPSIGNAL", "USER", "VOLUME", "WORKDIR"}) { + return errors.Errorf("invalid syntax for --change ", change) + } } } - history := []v1.History{ - {Comment: c.String("message")}, - } - - config := v1.Image{ - Config: changes, - History: history, - Author: c.String("author"), - } - if !c.Bool("quiet") { writer = os.Stderr } - ctr, err := runtime.LookupContainer(container) if err != nil { return errors.Wrapf(err, "error looking up container %q", container) } - newImage, err := ctr.Commit(c.BoolT("pause"), reference, writer, image.SigningOptions{}, config) - if err == nil { - fmt.Println(newImage.ID()) + + sc := image.GetSystemContext(runtime.GetConfig().SignaturePolicyPath, "", false) + coptions := buildah.CommitOptions{ + SignaturePolicyPath: runtime.GetConfig().SignaturePolicyPath, + ReportWriter: writer, + SystemContext: sc, + } + options := libpod.ContainerCommitOptions{ + CommitOptions: coptions, + Pause: c.Bool("pause"), + Message: c.String("message"), + Changes: c.StringSlice("change"), + Author: c.String("author"), + } + newImage, err := ctr.Commit(reference, options) + if err != nil { + return err } + fmt.Println(newImage.ID()) return nil } diff --git a/docs/podman-commit.1.md b/docs/podman-commit.1.md index 31eb8800a..9261a311a 100644 --- a/docs/podman-commit.1.md +++ b/docs/podman-commit.1.md @@ -6,13 +6,7 @@ podman commit - Create new image based on the changed container ## SYNOPSIS -**podman commit** -**TARBALL** -[**--author**|**-a**] -[**--change**|**-c**] -[**--message**|**-m**] -[**--help**|**-h**] -[**--verbose**] +**podman commit** [*options* [...]] CONTAINER IMAGE ## DESCRIPTION **podman commit** creates an image based on a changed container. The author of the @@ -23,12 +17,6 @@ committed. This minimizes the likelihood of data corruption when creating the ne image. If this is not desired, the **--pause** flag can be set to false. When the commit is complete, podman will print out the ID of the new image. -**podman [GLOBAL OPTIONS]** - -**podman commit [GLOBAL OPTIONS]** - -**podman commit [OPTIONS] CONTAINER** - ## OPTIONS **--author, -a** @@ -68,7 +56,7 @@ e3ce4d93051ceea088d1c242624d659be32cf1667ef62f1d16d6b60193e2c7a8 ``` ``` -# podman commit -q --author "firstName lastName" reverent_golick +# podman commit -q --author "firstName lastName" reverent_golick image-commited e3ce4d93051ceea088d1c242624d659be32cf1667ef62f1d16d6b60193e2c7a8 ``` diff --git a/libpod/buildah/buildah.go b/libpod/buildah/buildah.go new file mode 100644 index 000000000..b560f99a1 --- /dev/null +++ b/libpod/buildah/buildah.go @@ -0,0 +1,232 @@ +package buildah + +import ( + "encoding/json" + "path/filepath" + + is "github.com/containers/image/storage" + "github.com/containers/image/types" + "github.com/containers/storage" + "github.com/containers/storage/pkg/ioutils" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/projectatomic/libpod/cmd/podman/docker" +) + +const ( + // Package is the name of this package, used in help output and to + // identify working containers. + Package = "buildah" + // Version for the Package. Bump version in contrib/rpm/buildah.spec + // too. + Version = "0.15" + // The value we use to identify what type of information, currently a + // serialized Builder structure, we are using as per-container state. + // This should only be changed when we make incompatible changes to + // that data structure, as it's used to distinguish containers which + // are "ours" from ones that aren't. + containerType = Package + " 0.0.1" + // The file in the per-container directory which we use to store our + // per-container state. If it isn't there, then the container isn't + // one of our build containers. + stateFile = Package + ".json" +) + +// Builder objects are used to represent containers which are being used to +// build images. They also carry potential updates which will be applied to +// the image's configuration when the container's contents are used to build an +// image. +type Builder struct { + store storage.Store + + // Type is used to help identify a build container's metadata. It + // should not be modified. + Type string `json:"type"` + // FromImage is the name of the source image which was used to create + // the container, if one was used. It should not be modified. + FromImage string `json:"image,omitempty"` + // FromImageID is the ID of the source image which was used to create + // the container, if one was used. It should not be modified. + FromImageID string `json:"image-id"` + // Config is the source image's configuration. It should not be + // modified. + Config []byte `json:"config,omitempty"` + // Manifest is the source image's manifest. It should not be modified. + Manifest []byte `json:"manifest,omitempty"` + + // Container is the name of the build container. It should not be modified. + Container string `json:"container-name,omitempty"` + // ContainerID is the ID of the build container. It should not be modified. + ContainerID string `json:"container-id,omitempty"` + // MountPoint is the last location where the container's root + // filesystem was mounted. It should not be modified. + MountPoint string `json:"mountpoint,omitempty"` + // ProcessLabel is the SELinux process label associated with the container + ProcessLabel string `json:"process-label,omitempty"` + // MountLabel is the SELinux mount label associated with the container + MountLabel string `json:"mount-label,omitempty"` + + // ImageAnnotations is a set of key-value pairs which is stored in the + // image's manifest. + ImageAnnotations map[string]string `json:"annotations,omitempty"` + // ImageCreatedBy is a description of how this container was built. + ImageCreatedBy string `json:"created-by,omitempty"` + + // Image metadata and runtime settings, in multiple formats. + OCIv1 v1.Image `json:"ociv1,omitempty"` + Docker docker.V2Image `json:"docker,omitempty"` + // DefaultMountsFilePath is the file path holding the mounts to be mounted in "host-path:container-path" format + DefaultMountsFilePath string `json:"defaultMountsFilePath,omitempty"` + CommonBuildOpts *CommonBuildOptions +} + +// CommonBuildOptions are reseources that can be defined by flags for both buildah from and bud +type CommonBuildOptions struct { + // AddHost is the list of hostnames to add to the resolv.conf + AddHost []string + //CgroupParent it the path to cgroups under which the cgroup for the container will be created. + CgroupParent string + //CPUPeriod limits the CPU CFS (Completely Fair Scheduler) period + CPUPeriod uint64 + //CPUQuota limits the CPU CFS (Completely Fair Scheduler) quota + CPUQuota int64 + //CPUShares (relative weight + CPUShares uint64 + //CPUSetCPUs in which to allow execution (0-3, 0,1) + CPUSetCPUs string + //CPUSetMems memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. + CPUSetMems string + //Memory limit + Memory int64 + //MemorySwap limit value equal to memory plus swap. + MemorySwap int64 + //SecruityOpts modify the way container security is running + LabelOpts []string + SeccompProfilePath string + ApparmorProfile string + //ShmSize is the shared memory size + ShmSize string + //Ulimit options + Ulimit []string + //Volumes to bind mount into the container + Volumes []string +} + +// ImportOptions are used to initialize a Builder from an existing container +// which was created elsewhere. +type ImportOptions struct { + // Container is the name of the build container. + Container string + // SignaturePolicyPath specifies an override location for the signature + // policy which should be used for verifying the new image as it is + // being written. Except in specific circumstances, no value should be + // specified, indicating that the shared, system-wide default policy + // should be used. + SignaturePolicyPath string +} + +// ImportBuilder creates a new build configuration using an already-present +// container. +func ImportBuilder(store storage.Store, options ImportOptions) (*Builder, error) { + return importBuilder(store, options) +} + +func importBuilder(store storage.Store, options ImportOptions) (*Builder, error) { + if options.Container == "" { + return nil, errors.Errorf("container name must be specified") + } + + c, err := store.Container(options.Container) + if err != nil { + return nil, err + } + + systemContext := getSystemContext(&types.SystemContext{}, options.SignaturePolicyPath) + + builder, err := importBuilderDataFromImage(store, systemContext, c.ImageID, options.Container, c.ID) + if err != nil { + return nil, err + } + + if builder.FromImageID != "" { + if d, err2 := digest.Parse(builder.FromImageID); err2 == nil { + builder.Docker.Parent = docker.ID(d) + } else { + builder.Docker.Parent = docker.ID(digest.NewDigestFromHex(digest.Canonical.String(), builder.FromImageID)) + } + } + if builder.FromImage != "" { + builder.Docker.ContainerConfig.Image = builder.FromImage + } + + err = builder.Save() + if err != nil { + return nil, errors.Wrapf(err, "error saving builder state") + } + + return builder, nil +} + +func importBuilderDataFromImage(store storage.Store, systemContext *types.SystemContext, imageID, containerName, containerID string) (*Builder, error) { + manifest := []byte{} + config := []byte{} + imageName := "" + + if imageID != "" { + ref, err := is.Transport.ParseStoreReference(store, imageID) + if err != nil { + return nil, errors.Wrapf(err, "no such image %q", imageID) + } + src, err2 := ref.NewImage(systemContext) + if err2 != nil { + return nil, errors.Wrapf(err2, "error instantiating image") + } + defer src.Close() + config, err = src.ConfigBlob() + if err != nil { + return nil, errors.Wrapf(err, "error reading image configuration") + } + manifest, _, err = src.Manifest() + if err != nil { + return nil, errors.Wrapf(err, "error reading image manifest") + } + if img, err3 := store.Image(imageID); err3 == nil { + if len(img.Names) > 0 { + imageName = img.Names[0] + } + } + } + + builder := &Builder{ + store: store, + Type: containerType, + FromImage: imageName, + FromImageID: imageID, + Config: config, + Manifest: manifest, + Container: containerName, + ContainerID: containerID, + ImageAnnotations: map[string]string{}, + ImageCreatedBy: "", + } + + builder.initConfig() + + return builder, nil +} + +// Save saves the builder's current state to the build container's metadata. +// This should not need to be called directly, as other methods of the Builder +// object take care of saving their state. +func (b *Builder) Save() error { + buildstate, err := json.Marshal(b) + if err != nil { + return err + } + cdir, err := b.store.ContainerDirectory(b.ContainerID) + if err != nil { + return err + } + return ioutils.AtomicWriteFile(filepath.Join(cdir, stateFile), buildstate, 0600) +} diff --git a/libpod/buildah/commit.go b/libpod/buildah/commit.go new file mode 100644 index 000000000..b862031d2 --- /dev/null +++ b/libpod/buildah/commit.go @@ -0,0 +1,147 @@ +package buildah + +import ( + "fmt" + "io" + "time" + + cp "github.com/containers/image/copy" + "github.com/containers/image/signature" + is "github.com/containers/image/storage" + "github.com/containers/image/transports" + "github.com/containers/image/types" + "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// CommitOptions can be used to alter how an image is committed. +type CommitOptions struct { + // PreferredManifestType is the preferred type of image manifest. The + // image configuration format will be of a compatible type. + PreferredManifestType string + // Compression specifies the type of compression which is applied to + // layer blobs. The default is to not use compression, but + // archive.Gzip is recommended. + Compression archive.Compression + // SignaturePolicyPath specifies an override location for the signature + // policy which should be used for verifying the new image as it is + // being written. Except in specific circumstances, no value should be + // specified, indicating that the shared, system-wide default policy + // should be used. + SignaturePolicyPath string + // AdditionalTags is a list of additional names to add to the image, if + // the transport to which we're writing the image gives us a way to add + // them. + AdditionalTags []string + // ReportWriter is an io.Writer which will be used to log the writing + // of the new image. + ReportWriter io.Writer + // HistoryTimestamp is the timestamp used when creating new items in the + // image's history. If unset, the current time will be used. + HistoryTimestamp *time.Time + // github.com/containers/image/types SystemContext to hold credentials + // and other authentication/authorization information. + SystemContext *types.SystemContext +} + +// PushOptions can be used to alter how an image is copied somewhere. +type PushOptions struct { + // Compression specifies the type of compression which is applied to + // layer blobs. The default is to not use compression, but + // archive.Gzip is recommended. + Compression archive.Compression + // SignaturePolicyPath specifies an override location for the signature + // policy which should be used for verifying the new image as it is + // being written. Except in specific circumstances, no value should be + // specified, indicating that the shared, system-wide default policy + // should be used. + SignaturePolicyPath string + // ReportWriter is an io.Writer which will be used to log the writing + // of the new image. + ReportWriter io.Writer + // Store is the local storage store which holds the source image. + Store storage.Store + // github.com/containers/image/types SystemContext to hold credentials + // and other authentication/authorization information. + SystemContext *types.SystemContext + // ManifestType is the format to use when saving the imge using the 'dir' transport + // possible options are oci, v2s1, and v2s2 + ManifestType string +} + +// Commit writes the contents of the container, along with its updated +// configuration, to a new image in the specified location, and if we know how, +// add any additional tags that were specified. +func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error { + policy, err := signature.DefaultPolicy(getSystemContext(options.SystemContext, options.SignaturePolicyPath)) + if err != nil { + return errors.Wrapf(err, "error obtaining default signature policy") + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return errors.Wrapf(err, "error creating new signature policy context") + } + defer func() { + if err2 := policyContext.Destroy(); err2 != nil { + logrus.Debugf("error destroying signature policy context: %v", err2) + } + }() + // Check if we're keeping everything in local storage. If so, we can take certain shortcuts. + _, destIsStorage := dest.Transport().(is.StoreTransport) + exporting := !destIsStorage + src, err := b.makeImageRef(options.PreferredManifestType, exporting, options.Compression, options.HistoryTimestamp) + if err != nil { + return errors.Wrapf(err, "error computing layer digests and building metadata") + } + // "Copy" our image to where it needs to be. + err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter, nil, options.SystemContext, "")) + if err != nil { + return errors.Wrapf(err, "error copying layers and metadata") + } + if len(options.AdditionalTags) > 0 { + switch dest.Transport().Name() { + case is.Transport.Name(): + img, err := is.Transport.GetStoreImage(b.store, dest) + if err != nil { + return errors.Wrapf(err, "error locating just-written image %q", transports.ImageName(dest)) + } + err = AddImageNames(b.store, img, options.AdditionalTags) + if err != nil { + return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...)) + } + logrus.Debugf("assigned names %v to image %q", img.Names, img.ID) + default: + logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name()) + } + } + return nil +} + +// Push copies the contents of the image to a new location. +func Push(image string, dest types.ImageReference, options PushOptions) error { + systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath) + policy, err := signature.DefaultPolicy(systemContext) + if err != nil { + return errors.Wrapf(err, "error obtaining default signature policy") + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return errors.Wrapf(err, "error creating new signature policy context") + } + // Look up the image. + src, err := is.Transport.ParseStoreReference(options.Store, image) + if err != nil { + return errors.Wrapf(err, "error parsing reference to image %q", image) + } + // Copy everything. + err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter, nil, options.SystemContext, options.ManifestType)) + if err != nil { + return errors.Wrapf(err, "error copying layers and metadata") + } + if options.ReportWriter != nil { + fmt.Fprintf(options.ReportWriter, "\n") + } + return nil +} diff --git a/libpod/buildah/common.go b/libpod/buildah/common.go new file mode 100644 index 000000000..18c960003 --- /dev/null +++ b/libpod/buildah/common.go @@ -0,0 +1,28 @@ +package buildah + +import ( + "io" + + cp "github.com/containers/image/copy" + "github.com/containers/image/types" +) + +func getCopyOptions(reportWriter io.Writer, sourceSystemContext *types.SystemContext, destinationSystemContext *types.SystemContext, manifestType string) *cp.Options { + return &cp.Options{ + ReportWriter: reportWriter, + SourceCtx: sourceSystemContext, + DestinationCtx: destinationSystemContext, + ForceManifestMIMEType: manifestType, + } +} + +func getSystemContext(defaults *types.SystemContext, signaturePolicyPath string) *types.SystemContext { + sc := &types.SystemContext{} + if defaults != nil { + *sc = *defaults + } + if signaturePolicyPath != "" { + sc.SignaturePolicyPath = signaturePolicyPath + } + return sc +} diff --git a/libpod/buildah/config.go b/libpod/buildah/config.go new file mode 100644 index 000000000..0759ca9da --- /dev/null +++ b/libpod/buildah/config.go @@ -0,0 +1,607 @@ +package buildah + +import ( + "encoding/json" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/opencontainers/go-digest" + ociv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/projectatomic/libpod/cmd/podman/docker" +) + +// makeOCIv1Image builds the best OCIv1 image structure we can from the +// contents of the docker image structure. +func makeOCIv1Image(dimage *docker.V2Image) (ociv1.Image, error) { + config := dimage.Config + if config == nil { + config = &dimage.ContainerConfig + } + dcreated := dimage.Created.UTC() + image := ociv1.Image{ + Created: &dcreated, + Author: dimage.Author, + Architecture: dimage.Architecture, + OS: dimage.OS, + Config: ociv1.ImageConfig{ + User: config.User, + ExposedPorts: map[string]struct{}{}, + Env: config.Env, + Entrypoint: config.Entrypoint, + Cmd: config.Cmd, + Volumes: config.Volumes, + WorkingDir: config.WorkingDir, + Labels: config.Labels, + }, + RootFS: ociv1.RootFS{ + Type: "", + DiffIDs: []digest.Digest{}, + }, + History: []ociv1.History{}, + } + for port, what := range config.ExposedPorts { + image.Config.ExposedPorts[string(port)] = what + } + RootFS := docker.V2S2RootFS{} + if dimage.RootFS != nil { + RootFS = *dimage.RootFS + } + if RootFS.Type == docker.TypeLayers { + image.RootFS.Type = docker.TypeLayers + image.RootFS.DiffIDs = append(image.RootFS.DiffIDs, RootFS.DiffIDs...) + } + for _, history := range dimage.History { + hcreated := history.Created.UTC() + ohistory := ociv1.History{ + Created: &hcreated, + CreatedBy: history.CreatedBy, + Author: history.Author, + Comment: history.Comment, + EmptyLayer: history.EmptyLayer, + } + image.History = append(image.History, ohistory) + } + return image, nil +} + +// makeDockerV2S2Image builds the best docker image structure we can from the +// contents of the OCI image structure. +func makeDockerV2S2Image(oimage *ociv1.Image) (docker.V2Image, error) { + image := docker.V2Image{ + V1Image: docker.V1Image{Created: oimage.Created.UTC(), + Author: oimage.Author, + Architecture: oimage.Architecture, + OS: oimage.OS, + ContainerConfig: docker.Config{ + User: oimage.Config.User, + ExposedPorts: docker.PortSet{}, + Env: oimage.Config.Env, + Entrypoint: oimage.Config.Entrypoint, + Cmd: oimage.Config.Cmd, + Volumes: oimage.Config.Volumes, + WorkingDir: oimage.Config.WorkingDir, + Labels: oimage.Config.Labels, + }, + }, + RootFS: &docker.V2S2RootFS{ + Type: "", + DiffIDs: []digest.Digest{}, + }, + History: []docker.V2S2History{}, + } + for port, what := range oimage.Config.ExposedPorts { + image.ContainerConfig.ExposedPorts[docker.Port(port)] = what + } + if oimage.RootFS.Type == docker.TypeLayers { + image.RootFS.Type = docker.TypeLayers + image.RootFS.DiffIDs = append(image.RootFS.DiffIDs, oimage.RootFS.DiffIDs...) + } + for _, history := range oimage.History { + dhistory := docker.V2S2History{ + Created: history.Created.UTC(), + CreatedBy: history.CreatedBy, + Author: history.Author, + Comment: history.Comment, + EmptyLayer: history.EmptyLayer, + } + image.History = append(image.History, dhistory) + } + image.Config = &image.ContainerConfig + return image, nil +} + +// makeDockerV2S1Image builds the best docker image structure we can from the +// contents of the V2S1 image structure. +func makeDockerV2S1Image(manifest docker.V2S1Manifest) (docker.V2Image, error) { + // Treat the most recent (first) item in the history as a description of the image. + if len(manifest.History) == 0 { + return docker.V2Image{}, errors.Errorf("error parsing image configuration from manifest") + } + dimage := docker.V2Image{} + err := json.Unmarshal([]byte(manifest.History[0].V1Compatibility), &dimage) + if err != nil { + return docker.V2Image{}, err + } + if dimage.DockerVersion == "" { + return docker.V2Image{}, errors.Errorf("error parsing image configuration from history") + } + // The DiffID list is intended to contain the sums of _uncompressed_ blobs, and these are most + // likely compressed, so leave the list empty to avoid potential confusion later on. We can + // construct a list with the correct values when we prep layers for pushing, so we don't lose. + // information by leaving this part undone. + rootFS := &docker.V2S2RootFS{ + Type: docker.TypeLayers, + DiffIDs: []digest.Digest{}, + } + // Build a filesystem history. + history := []docker.V2S2History{} + lastID := "" + for i := range manifest.History { + // Decode the compatibility field. + dcompat := docker.V1Compatibility{} + if err = json.Unmarshal([]byte(manifest.History[i].V1Compatibility), &dcompat); err != nil { + return docker.V2Image{}, errors.Errorf("error parsing image compatibility data (%q) from history", manifest.History[i].V1Compatibility) + } + // Skip this history item if it shares the ID of the last one + // that we saw, since the image library will do the same. + if i > 0 && dcompat.ID == lastID { + continue + } + lastID = dcompat.ID + // Construct a new history item using the recovered information. + createdBy := "" + if len(dcompat.ContainerConfig.Cmd) > 0 { + createdBy = strings.Join(dcompat.ContainerConfig.Cmd, " ") + } + h := docker.V2S2History{ + Created: dcompat.Created.UTC(), + Author: dcompat.Author, + CreatedBy: createdBy, + Comment: dcompat.Comment, + EmptyLayer: dcompat.ThrowAway, + } + // Prepend this layer to the list, because a v2s1 format manifest's list is in reverse order + // compared to v2s2, which lists earlier layers before later ones. + history = append([]docker.V2S2History{h}, history...) + } + dimage.RootFS = rootFS + dimage.History = history + return dimage, nil +} + +func (b *Builder) initConfig() { + image := ociv1.Image{} + dimage := docker.V2Image{} + if len(b.Config) > 0 { + // Try to parse the image configuration. If we fail start over from scratch. + if err := json.Unmarshal(b.Config, &dimage); err == nil && dimage.DockerVersion != "" { + if image, err = makeOCIv1Image(&dimage); err != nil { + image = ociv1.Image{} + } + } else { + if err := json.Unmarshal(b.Config, &image); err != nil { + if dimage, err = makeDockerV2S2Image(&image); err != nil { + dimage = docker.V2Image{} + } + } + } + b.OCIv1 = image + b.Docker = dimage + } else { + // Try to dig out the image configuration from the manifest. + manifest := docker.V2S1Manifest{} + if err := json.Unmarshal(b.Manifest, &manifest); err == nil && manifest.SchemaVersion == 1 { + if dimage, err = makeDockerV2S1Image(manifest); err == nil { + if image, err = makeOCIv1Image(&dimage); err != nil { + image = ociv1.Image{} + } + } + } + b.OCIv1 = image + b.Docker = dimage + } + if len(b.Manifest) > 0 { + // Attempt to recover format-specific data from the manifest. + v1Manifest := ociv1.Manifest{} + if json.Unmarshal(b.Manifest, &v1Manifest) == nil { + b.ImageAnnotations = v1Manifest.Annotations + } + } + b.fixupConfig() +} + +func (b *Builder) fixupConfig() { + if b.Docker.Config != nil { + // Prefer image-level settings over those from the container it was built from. + b.Docker.ContainerConfig = *b.Docker.Config + } + b.Docker.Config = &b.Docker.ContainerConfig + b.Docker.DockerVersion = "" + now := time.Now().UTC() + if b.Docker.Created.IsZero() { + b.Docker.Created = now + } + if b.OCIv1.Created == nil || b.OCIv1.Created.IsZero() { + b.OCIv1.Created = &now + } + if b.OS() == "" { + b.SetOS(runtime.GOOS) + } + if b.Architecture() == "" { + b.SetArchitecture(runtime.GOARCH) + } + if b.WorkDir() == "" { + b.SetWorkDir(string(filepath.Separator)) + } +} + +// Annotations returns a set of key-value pairs from the image's manifest. +func (b *Builder) Annotations() map[string]string { + return copyStringStringMap(b.ImageAnnotations) +} + +// SetAnnotation adds or overwrites a key's value from the image's manifest. +// Note: this setting is not present in the Docker v2 image format, so it is +// discarded when writing images using Docker v2 formats. +func (b *Builder) SetAnnotation(key, value string) { + if b.ImageAnnotations == nil { + b.ImageAnnotations = map[string]string{} + } + b.ImageAnnotations[key] = value +} + +// UnsetAnnotation removes a key and its value from the image's manifest, if +// it's present. +func (b *Builder) UnsetAnnotation(key string) { + delete(b.ImageAnnotations, key) +} + +// ClearAnnotations removes all keys and their values from the image's +// manifest. +func (b *Builder) ClearAnnotations() { + b.ImageAnnotations = map[string]string{} +} + +// CreatedBy returns a description of how this image was built. +func (b *Builder) CreatedBy() string { + return b.ImageCreatedBy +} + +// SetCreatedBy sets the description of how this image was built. +func (b *Builder) SetCreatedBy(how string) { + b.ImageCreatedBy = how +} + +// OS returns a name of the OS on which the container, or a container built +// using an image built from this container, is intended to be run. +func (b *Builder) OS() string { + return b.OCIv1.OS +} + +// SetOS sets the name of the OS on which the container, or a container built +// using an image built from this container, is intended to be run. +func (b *Builder) SetOS(os string) { + b.OCIv1.OS = os + b.Docker.OS = os +} + +// Architecture returns a name of the architecture on which the container, or a +// container built using an image built from this container, is intended to be +// run. +func (b *Builder) Architecture() string { + return b.OCIv1.Architecture +} + +// SetArchitecture sets the name of the architecture on which the container, or +// a container built using an image built from this container, is intended to +// be run. +func (b *Builder) SetArchitecture(arch string) { + b.OCIv1.Architecture = arch + b.Docker.Architecture = arch +} + +// Maintainer returns contact information for the person who built the image. +func (b *Builder) Maintainer() string { + return b.OCIv1.Author +} + +// SetMaintainer sets contact information for the person who built the image. +func (b *Builder) SetMaintainer(who string) { + b.OCIv1.Author = who + b.Docker.Author = who +} + +// User returns information about the user as whom the container, or a +// container built using an image built from this container, should be run. +func (b *Builder) User() string { + return b.OCIv1.Config.User +} + +// SetUser sets information about the user as whom the container, or a +// container built using an image built from this container, should be run. +// Acceptable forms are a user name or ID, optionally followed by a colon and a +// group name or ID. +func (b *Builder) SetUser(spec string) { + b.OCIv1.Config.User = spec + b.Docker.Config.User = spec +} + +// WorkDir returns the default working directory for running commands in the +// container, or in a container built using an image built from this container. +func (b *Builder) WorkDir() string { + return b.OCIv1.Config.WorkingDir +} + +// SetWorkDir sets the location of the default working directory for running +// commands in the container, or in a container built using an image built from +// this container. +func (b *Builder) SetWorkDir(there string) { + b.OCIv1.Config.WorkingDir = there + b.Docker.Config.WorkingDir = there +} + +// Shell returns the default shell for running commands in the +// container, or in a container built using an image built from this container. +func (b *Builder) Shell() []string { + return b.Docker.Config.Shell +} + +// SetShell sets the default shell for running +// commands in the container, or in a container built using an image built from +// this container. +// Note: this setting is not present in the OCIv1 image format, so it is +// discarded when writing images using OCIv1 formats. +func (b *Builder) SetShell(shell []string) { + b.Docker.Config.Shell = shell +} + +// Env returns a list of key-value pairs to be set when running commands in the +// container, or in a container built using an image built from this container. +func (b *Builder) Env() []string { + return copyStringSlice(b.OCIv1.Config.Env) +} + +// SetEnv adds or overwrites a value to the set of environment strings which +// should be set when running commands in the container, or in a container +// built using an image built from this container. +func (b *Builder) SetEnv(k string, v string) { + reset := func(s *[]string) { + n := []string{} + for i := range *s { + if !strings.HasPrefix((*s)[i], k+"=") { + n = append(n, (*s)[i]) + } + } + n = append(n, k+"="+v) + *s = n + } + reset(&b.OCIv1.Config.Env) + reset(&b.Docker.Config.Env) +} + +// UnsetEnv removes a value from the set of environment strings which should be +// set when running commands in this container, or in a container built using +// an image built from this container. +func (b *Builder) UnsetEnv(k string) { + unset := func(s *[]string) { + n := []string{} + for i := range *s { + if !strings.HasPrefix((*s)[i], k+"=") { + n = append(n, (*s)[i]) + } + } + *s = n + } + unset(&b.OCIv1.Config.Env) + unset(&b.Docker.Config.Env) +} + +// ClearEnv removes all values from the set of environment strings which should +// be set when running commands in this container, or in a container built +// using an image built from this container. +func (b *Builder) ClearEnv() { + b.OCIv1.Config.Env = []string{} + b.Docker.Config.Env = []string{} +} + +// Cmd returns the default command, or command parameters if an Entrypoint is +// set, to use when running a container built from an image built from this +// container. +func (b *Builder) Cmd() []string { + return copyStringSlice(b.OCIv1.Config.Cmd) +} + +// SetCmd sets the default command, or command parameters if an Entrypoint is +// set, to use when running a container built from an image built from this +// container. +func (b *Builder) SetCmd(cmd []string) { + b.OCIv1.Config.Cmd = copyStringSlice(cmd) + b.Docker.Config.Cmd = copyStringSlice(cmd) +} + +// Entrypoint returns the command to be run for containers built from images +// built from this container. +func (b *Builder) Entrypoint() []string { + return copyStringSlice(b.OCIv1.Config.Entrypoint) +} + +// SetEntrypoint sets the command to be run for in containers built from images +// built from this container. +func (b *Builder) SetEntrypoint(ep []string) { + b.OCIv1.Config.Entrypoint = copyStringSlice(ep) + b.Docker.Config.Entrypoint = copyStringSlice(ep) +} + +// Labels returns a set of key-value pairs from the image's runtime +// configuration. +func (b *Builder) Labels() map[string]string { + return copyStringStringMap(b.OCIv1.Config.Labels) +} + +// SetLabel adds or overwrites a key's value from the image's runtime +// configuration. +func (b *Builder) SetLabel(k string, v string) { + if b.OCIv1.Config.Labels == nil { + b.OCIv1.Config.Labels = map[string]string{} + } + b.OCIv1.Config.Labels[k] = v + if b.Docker.Config.Labels == nil { + b.Docker.Config.Labels = map[string]string{} + } + b.Docker.Config.Labels[k] = v +} + +// UnsetLabel removes a key and its value from the image's runtime +// configuration, if it's present. +func (b *Builder) UnsetLabel(k string) { + delete(b.OCIv1.Config.Labels, k) + delete(b.Docker.Config.Labels, k) +} + +// ClearLabels removes all keys and their values from the image's runtime +// configuration. +func (b *Builder) ClearLabels() { + b.OCIv1.Config.Labels = map[string]string{} + b.Docker.Config.Labels = map[string]string{} +} + +// Ports returns the set of ports which should be exposed when a container +// based on an image built from this container is run. +func (b *Builder) Ports() []string { + p := []string{} + for k := range b.OCIv1.Config.ExposedPorts { + p = append(p, k) + } + return p +} + +// SetPort adds or overwrites an exported port in the set of ports which should +// be exposed when a container based on an image built from this container is +// run. +func (b *Builder) SetPort(p string) { + if b.OCIv1.Config.ExposedPorts == nil { + b.OCIv1.Config.ExposedPorts = map[string]struct{}{} + } + b.OCIv1.Config.ExposedPorts[p] = struct{}{} + if b.Docker.Config.ExposedPorts == nil { + b.Docker.Config.ExposedPorts = make(docker.PortSet) + } + b.Docker.Config.ExposedPorts[docker.Port(p)] = struct{}{} +} + +// UnsetPort removes an exposed port from the set of ports which should be +// exposed when a container based on an image built from this container is run. +func (b *Builder) UnsetPort(p string) { + delete(b.OCIv1.Config.ExposedPorts, p) + delete(b.Docker.Config.ExposedPorts, docker.Port(p)) +} + +// ClearPorts empties the set of ports which should be exposed when a container +// based on an image built from this container is run. +func (b *Builder) ClearPorts() { + b.OCIv1.Config.ExposedPorts = map[string]struct{}{} + b.Docker.Config.ExposedPorts = docker.PortSet{} +} + +// Volumes returns a list of filesystem locations which should be mounted from +// outside of the container when a container built from an image built from +// this container is run. +func (b *Builder) Volumes() []string { + v := []string{} + for k := range b.OCIv1.Config.Volumes { + v = append(v, k) + } + return v +} + +// AddVolume adds a location to the image's list of locations which should be +// mounted from outside of the container when a container based on an image +// built from this container is run. +func (b *Builder) AddVolume(v string) { + if b.OCIv1.Config.Volumes == nil { + b.OCIv1.Config.Volumes = map[string]struct{}{} + } + b.OCIv1.Config.Volumes[v] = struct{}{} + if b.Docker.Config.Volumes == nil { + b.Docker.Config.Volumes = map[string]struct{}{} + } + b.Docker.Config.Volumes[v] = struct{}{} +} + +// RemoveVolume removes a location from the list of locations which should be +// mounted from outside of the container when a container based on an image +// built from this container is run. +func (b *Builder) RemoveVolume(v string) { + delete(b.OCIv1.Config.Volumes, v) + delete(b.Docker.Config.Volumes, v) +} + +// ClearVolumes removes all locations from the image's list of locations which +// should be mounted from outside of the container when a container based on an +// image built from this container is run. +func (b *Builder) ClearVolumes() { + b.OCIv1.Config.Volumes = map[string]struct{}{} + b.Docker.Config.Volumes = map[string]struct{}{} +} + +// Hostname returns the hostname which will be set in the container and in +// containers built using images built from the container. +func (b *Builder) Hostname() string { + return b.Docker.Config.Hostname +} + +// SetHostname sets the hostname which will be set in the container and in +// containers built using images built from the container. +// Note: this setting is not present in the OCIv1 image format, so it is +// discarded when writing images using OCIv1 formats. +func (b *Builder) SetHostname(name string) { + b.Docker.Config.Hostname = name +} + +// Domainname returns the domainname which will be set in the container and in +// containers built using images built from the container. +func (b *Builder) Domainname() string { + return b.Docker.Config.Domainname +} + +// SetDomainname sets the domainname which will be set in the container and in +// containers built using images built from the container. +// Note: this setting is not present in the OCIv1 image format, so it is +// discarded when writing images using OCIv1 formats. +func (b *Builder) SetDomainname(name string) { + b.Docker.Config.Domainname = name +} + +// SetDefaultMountsFilePath sets the mounts file path for testing purposes +func (b *Builder) SetDefaultMountsFilePath(path string) { + b.DefaultMountsFilePath = path +} + +// Comment returns the comment which will be set in the container and in +//containers built using images buiilt from the container +func (b *Builder) Comment() string { + return b.Docker.Comment +} + +// SetComment sets the Comment which will be set in the container and in +// containers built using images built from the container. +func (b *Builder) SetComment(comment string) { + b.Docker.Comment = comment + b.OCIv1.History[0].Comment = comment +} + +// StopSignal returns the signal which will be set in the container and in +//containers built using images buiilt from the container +func (b *Builder) StopSignal() string { + return b.Docker.Config.StopSignal +} + +// SetStopSignal sets the signal which will be set in the container and in +// containers built using images built from the container. +func (b *Builder) SetStopSignal(stopSignal string) { + b.OCIv1.Config.StopSignal = stopSignal + b.Docker.Config.StopSignal = stopSignal +} diff --git a/libpod/buildah/image.go b/libpod/buildah/image.go new file mode 100644 index 000000000..656c39201 --- /dev/null +++ b/libpod/buildah/image.go @@ -0,0 +1,529 @@ +package buildah + +import ( + "bytes" + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/containers/image/docker/reference" + "github.com/containers/image/image" + is "github.com/containers/image/storage" + "github.com/containers/image/types" + "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/ioutils" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/projectatomic/libpod/cmd/podman/docker" + "github.com/sirupsen/logrus" +) + +const ( + // OCIv1ImageManifest is the MIME type of an OCIv1 image manifest, + // suitable for specifying as a value of the PreferredManifestType + // member of a CommitOptions structure. It is also the default. + OCIv1ImageManifest = v1.MediaTypeImageManifest + // Dockerv2ImageManifest is the MIME type of a Docker v2s2 image + // manifest, suitable for specifying as a value of the + // PreferredManifestType member of a CommitOptions structure. + Dockerv2ImageManifest = docker.V2S2MediaTypeManifest +) + +type containerImageRef struct { + store storage.Store + compression archive.Compression + name reference.Named + names []string + layerID string + oconfig []byte + dconfig []byte + created time.Time + createdBy string + annotations map[string]string + preferredManifestType string + exporting bool +} + +type containerImageSource struct { + path string + ref *containerImageRef + store storage.Store + layerID string + names []string + compression archive.Compression + config []byte + configDigest digest.Digest + manifest []byte + manifestType string + exporting bool +} + +func (i *containerImageRef) NewImage(sc *types.SystemContext) (types.ImageCloser, error) { + src, err := i.NewImageSource(sc) + if err != nil { + return nil, err + } + return image.FromSource(sc, src) +} + +func expectedOCIDiffIDs(image v1.Image) int { + expected := 0 + for _, history := range image.History { + if !history.EmptyLayer { + expected = expected + 1 + } + } + return expected +} + +func expectedDockerDiffIDs(image docker.V2Image) int { + expected := 0 + for _, history := range image.History { + if !history.EmptyLayer { + expected = expected + 1 + } + } + return expected +} + +func (i *containerImageRef) NewImageSource(sc *types.SystemContext) (src types.ImageSource, err error) { + // Decide which type of manifest and configuration output we're going to provide. + manifestType := i.preferredManifestType + // If it's not a format we support, return an error. + if manifestType != v1.MediaTypeImageManifest && manifestType != docker.V2S2MediaTypeManifest { + return nil, errors.Errorf("no supported manifest types (attempted to use %q, only know %q and %q)", + manifestType, v1.MediaTypeImageManifest, docker.V2S2MediaTypeManifest) + } + // Start building the list of layers using the read-write layer. + layers := []string{} + layerID := i.layerID + layer, err := i.store.Layer(layerID) + if err != nil { + return nil, errors.Wrapf(err, "unable to read layer %q", layerID) + } + // Walk the list of parent layers, prepending each as we go. + for layer != nil { + layers = append(append([]string{}, layerID), layers...) + layerID = layer.Parent + if layerID == "" { + err = nil + break + } + layer, err = i.store.Layer(layerID) + if err != nil { + return nil, errors.Wrapf(err, "unable to read layer %q", layerID) + } + } + logrus.Debugf("layer list: %q", layers) + + // Make a temporary directory to hold blobs. + path, err := ioutil.TempDir(os.TempDir(), Package) + if err != nil { + return nil, err + } + logrus.Debugf("using %q to hold temporary data", path) + defer func() { + if src == nil { + err2 := os.RemoveAll(path) + if err2 != nil { + logrus.Errorf("error removing %q: %v", path, err) + } + } + }() + + // Build fresh copies of the configurations so that we don't mess with the values in the Builder + // object itself. + oimage := v1.Image{} + err = json.Unmarshal(i.oconfig, &oimage) + if err != nil { + return nil, err + } + created := i.created + oimage.Created = &created + dimage := docker.V2Image{} + err = json.Unmarshal(i.dconfig, &dimage) + if err != nil { + return nil, err + } + dimage.Created = created + + // Start building manifests. + omanifest := v1.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: v1.Descriptor{ + MediaType: v1.MediaTypeImageConfig, + }, + Layers: []v1.Descriptor{}, + Annotations: i.annotations, + } + dmanifest := docker.V2S2Manifest{ + V2Versioned: docker.V2Versioned{ + SchemaVersion: 2, + MediaType: docker.V2S2MediaTypeManifest, + }, + Config: docker.V2S2Descriptor{ + MediaType: docker.V2S2MediaTypeImageConfig, + }, + Layers: []docker.V2S2Descriptor{}, + } + + oimage.RootFS.Type = docker.TypeLayers + oimage.RootFS.DiffIDs = []digest.Digest{} + dimage.RootFS = &docker.V2S2RootFS{} + dimage.RootFS.Type = docker.TypeLayers + dimage.RootFS.DiffIDs = []digest.Digest{} + + // Extract each layer and compute its digests, both compressed (if requested) and uncompressed. + for _, layerID := range layers { + // The default layer media type assumes no compression. + omediaType := v1.MediaTypeImageLayer + dmediaType := docker.V2S2MediaTypeUncompressedLayer + // If we're not re-exporting the data, reuse the blobsum and diff IDs. + if !i.exporting && layerID != i.layerID { + layer, err2 := i.store.Layer(layerID) + if err2 != nil { + return nil, errors.Wrapf(err, "unable to locate layer %q", layerID) + } + if layer.UncompressedDigest == "" { + return nil, errors.Errorf("unable to look up size of layer %q", layerID) + } + layerBlobSum := layer.UncompressedDigest + layerBlobSize := layer.UncompressedSize + // Note this layer in the manifest, using the uncompressed blobsum. + olayerDescriptor := v1.Descriptor{ + MediaType: omediaType, + Digest: layerBlobSum, + Size: layerBlobSize, + } + omanifest.Layers = append(omanifest.Layers, olayerDescriptor) + dlayerDescriptor := docker.V2S2Descriptor{ + MediaType: dmediaType, + Digest: layerBlobSum, + Size: layerBlobSize, + } + dmanifest.Layers = append(dmanifest.Layers, dlayerDescriptor) + // Note this layer in the list of diffIDs, again using the uncompressed blobsum. + oimage.RootFS.DiffIDs = append(oimage.RootFS.DiffIDs, layerBlobSum) + dimage.RootFS.DiffIDs = append(dimage.RootFS.DiffIDs, layerBlobSum) + continue + } + // Figure out if we need to change the media type, in case we're using compression. + if i.compression != archive.Uncompressed { + switch i.compression { + case archive.Gzip: + omediaType = v1.MediaTypeImageLayerGzip + dmediaType = docker.V2S2MediaTypeLayer + logrus.Debugf("compressing layer %q with gzip", layerID) + case archive.Bzip2: + // Until the image specs define a media type for bzip2-compressed layers, even if we know + // how to decompress them, we can't try to compress layers with bzip2. + return nil, errors.New("media type for bzip2-compressed layers is not defined") + case archive.Xz: + // Until the image specs define a media type for xz-compressed layers, even if we know + // how to decompress them, we can't try to compress layers with xz. + return nil, errors.New("media type for xz-compressed layers is not defined") + default: + logrus.Debugf("compressing layer %q with unknown compressor(?)", layerID) + } + } + // Start reading the layer. + noCompression := archive.Uncompressed + diffOptions := &storage.DiffOptions{ + Compression: &noCompression, + } + rc, err := i.store.Diff("", layerID, diffOptions) + if err != nil { + return nil, errors.Wrapf(err, "error extracting layer %q", layerID) + } + defer rc.Close() + srcHasher := digest.Canonical.Digester() + reader := io.TeeReader(rc, srcHasher.Hash()) + // Set up to write the possibly-recompressed blob. + layerFile, err := os.OpenFile(filepath.Join(path, "layer"), os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + return nil, errors.Wrapf(err, "error opening file for layer %q", layerID) + } + destHasher := digest.Canonical.Digester() + counter := ioutils.NewWriteCounter(layerFile) + multiWriter := io.MultiWriter(counter, destHasher.Hash()) + // Compress the layer, if we're recompressing it. + writer, err := archive.CompressStream(multiWriter, i.compression) + if err != nil { + return nil, errors.Wrapf(err, "error compressing layer %q", layerID) + } + size, err := io.Copy(writer, reader) + if err != nil { + return nil, errors.Wrapf(err, "error storing layer %q to file", layerID) + } + writer.Close() + layerFile.Close() + if i.compression == archive.Uncompressed { + if size != counter.Count { + return nil, errors.Errorf("error storing layer %q to file: inconsistent layer size (copied %d, wrote %d)", layerID, size, counter.Count) + } + } else { + size = counter.Count + } + logrus.Debugf("layer %q size is %d bytes", layerID, size) + // Rename the layer so that we can more easily find it by digest later. + err = os.Rename(filepath.Join(path, "layer"), filepath.Join(path, destHasher.Digest().String())) + if err != nil { + return nil, errors.Wrapf(err, "error storing layer %q to file", layerID) + } + // Add a note in the manifest about the layer. The blobs are identified by their possibly- + // compressed blob digests. + olayerDescriptor := v1.Descriptor{ + MediaType: omediaType, + Digest: destHasher.Digest(), + Size: size, + } + omanifest.Layers = append(omanifest.Layers, olayerDescriptor) + dlayerDescriptor := docker.V2S2Descriptor{ + MediaType: dmediaType, + Digest: destHasher.Digest(), + Size: size, + } + dmanifest.Layers = append(dmanifest.Layers, dlayerDescriptor) + // Add a note about the diffID, which is always the layer's uncompressed digest. + oimage.RootFS.DiffIDs = append(oimage.RootFS.DiffIDs, srcHasher.Digest()) + dimage.RootFS.DiffIDs = append(dimage.RootFS.DiffIDs, srcHasher.Digest()) + } + + // Build history notes in the image configurations. + onews := v1.History{ + Created: &i.created, + CreatedBy: i.createdBy, + Author: oimage.Author, + EmptyLayer: false, + } + oimage.History = append(oimage.History, onews) + dnews := docker.V2S2History{ + Created: i.created, + CreatedBy: i.createdBy, + Author: dimage.Author, + EmptyLayer: false, + } + dimage.History = append(dimage.History, dnews) + + // Sanity check that we didn't just create a mismatch between non-empty layers in the + // history and the number of diffIDs. + expectedDiffIDs := expectedOCIDiffIDs(oimage) + if len(oimage.RootFS.DiffIDs) != expectedDiffIDs { + return nil, errors.Errorf("internal error: history lists %d non-empty layers, but we have %d layers on disk", expectedDiffIDs, len(oimage.RootFS.DiffIDs)) + } + expectedDiffIDs = expectedDockerDiffIDs(dimage) + if len(dimage.RootFS.DiffIDs) != expectedDiffIDs { + return nil, errors.Errorf("internal error: history lists %d non-empty layers, but we have %d layers on disk", expectedDiffIDs, len(dimage.RootFS.DiffIDs)) + } + + // Encode the image configuration blob. + oconfig, err := json.Marshal(&oimage) + if err != nil { + return nil, err + } + logrus.Debugf("OCIv1 config = %s", oconfig) + + // Add the configuration blob to the manifest. + omanifest.Config.Digest = digest.Canonical.FromBytes(oconfig) + omanifest.Config.Size = int64(len(oconfig)) + omanifest.Config.MediaType = v1.MediaTypeImageConfig + + // Encode the manifest. + omanifestbytes, err := json.Marshal(&omanifest) + if err != nil { + return nil, err + } + logrus.Debugf("OCIv1 manifest = %s", omanifestbytes) + + // Encode the image configuration blob. + dconfig, err := json.Marshal(&dimage) + if err != nil { + return nil, err + } + logrus.Debugf("Docker v2s2 config = %s", dconfig) + + // Add the configuration blob to the manifest. + dmanifest.Config.Digest = digest.Canonical.FromBytes(dconfig) + dmanifest.Config.Size = int64(len(dconfig)) + dmanifest.Config.MediaType = docker.V2S2MediaTypeImageConfig + + // Encode the manifest. + dmanifestbytes, err := json.Marshal(&dmanifest) + if err != nil { + return nil, err + } + logrus.Debugf("Docker v2s2 manifest = %s", dmanifestbytes) + + // Decide which manifest and configuration blobs we'll actually output. + var config []byte + var manifest []byte + switch manifestType { + case v1.MediaTypeImageManifest: + manifest = omanifestbytes + config = oconfig + case docker.V2S2MediaTypeManifest: + manifest = dmanifestbytes + config = dconfig + default: + panic("unreachable code: unsupported manifest type") + } + src = &containerImageSource{ + path: path, + ref: i, + store: i.store, + layerID: i.layerID, + names: i.names, + compression: i.compression, + config: config, + configDigest: digest.Canonical.FromBytes(config), + manifest: manifest, + manifestType: manifestType, + exporting: i.exporting, + } + return src, nil +} + +func (i *containerImageRef) NewImageDestination(sc *types.SystemContext) (types.ImageDestination, error) { + return nil, errors.Errorf("can't write to a container") +} + +func (i *containerImageRef) DockerReference() reference.Named { + return i.name +} + +func (i *containerImageRef) StringWithinTransport() string { + if len(i.names) > 0 { + return i.names[0] + } + return "" +} + +func (i *containerImageRef) DeleteImage(*types.SystemContext) error { + // we were never here + return nil +} + +func (i *containerImageRef) PolicyConfigurationIdentity() string { + return "" +} + +func (i *containerImageRef) PolicyConfigurationNamespaces() []string { + return nil +} + +func (i *containerImageRef) Transport() types.ImageTransport { + return is.Transport +} + +func (i *containerImageSource) Close() error { + err := os.RemoveAll(i.path) + if err != nil { + logrus.Errorf("error removing %q: %v", i.path, err) + } + return err +} + +func (i *containerImageSource) Reference() types.ImageReference { + return i.ref +} + +func (i *containerImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + if instanceDigest != nil && *instanceDigest != digest.FromBytes(i.manifest) { + return nil, errors.Errorf("TODO") + } + return nil, nil +} + +func (i *containerImageSource) GetManifest(instanceDigest *digest.Digest) ([]byte, string, error) { + if instanceDigest != nil && *instanceDigest != digest.FromBytes(i.manifest) { + return nil, "", errors.Errorf("TODO") + } + return i.manifest, i.manifestType, nil +} + +func (i *containerImageSource) LayerInfosForCopy() ([]types.BlobInfo, error) { + return nil, nil +} + +func (i *containerImageSource) GetBlob(blob types.BlobInfo) (reader io.ReadCloser, size int64, err error) { + if blob.Digest == i.configDigest { + logrus.Debugf("start reading config") + reader := bytes.NewReader(i.config) + closer := func() error { + logrus.Debugf("finished reading config") + return nil + } + return ioutils.NewReadCloserWrapper(reader, closer), reader.Size(), nil + } + layerFile, err := os.OpenFile(filepath.Join(i.path, blob.Digest.String()), os.O_RDONLY, 0600) + if err != nil { + logrus.Debugf("error reading layer %q: %v", blob.Digest.String(), err) + return nil, -1, err + } + size = -1 + st, err := layerFile.Stat() + if err != nil { + logrus.Warnf("error reading size of layer %q: %v", blob.Digest.String(), err) + } else { + size = st.Size() + } + logrus.Debugf("reading layer %q", blob.Digest.String()) + closer := func() error { + layerFile.Close() + logrus.Debugf("finished reading layer %q", blob.Digest.String()) + return nil + } + return ioutils.NewReadCloserWrapper(layerFile, closer), size, nil +} + +func (b *Builder) makeImageRef(manifestType string, exporting bool, compress archive.Compression, historyTimestamp *time.Time) (types.ImageReference, error) { + var name reference.Named + container, err := b.store.Container(b.ContainerID) + if err != nil { + return nil, errors.Wrapf(err, "error locating container %q", b.ContainerID) + } + if len(container.Names) > 0 { + if parsed, err2 := reference.ParseNamed(container.Names[0]); err2 == nil { + name = parsed + } + } + if manifestType == "" { + manifestType = OCIv1ImageManifest + } + oconfig, err := json.Marshal(&b.OCIv1) + if err != nil { + return nil, errors.Wrapf(err, "error encoding OCI-format image configuration") + } + dconfig, err := json.Marshal(&b.Docker) + if err != nil { + return nil, errors.Wrapf(err, "error encoding docker-format image configuration") + } + created := time.Now().UTC() + if historyTimestamp != nil { + created = historyTimestamp.UTC() + } + ref := &containerImageRef{ + store: b.store, + compression: compress, + name: name, + names: container.Names, + layerID: container.LayerID, + oconfig: oconfig, + dconfig: dconfig, + created: created, + createdBy: b.CreatedBy(), + annotations: b.Annotations(), + preferredManifestType: manifestType, + exporting: exporting, + } + return ref, nil +} diff --git a/libpod/buildah/util.go b/libpod/buildah/util.go new file mode 100644 index 000000000..96f9ebf86 --- /dev/null +++ b/libpod/buildah/util.go @@ -0,0 +1,67 @@ +package buildah + +import ( + "github.com/containers/image/docker/reference" + "github.com/containers/storage" + "github.com/containers/storage/pkg/reexec" + "github.com/pkg/errors" +) + +// InitReexec is a wrapper for reexec.Init(). It should be called at +// the start of main(), and if it returns true, main() should return +// immediately. +func InitReexec() bool { + return reexec.Init() +} + +func copyStringStringMap(m map[string]string) map[string]string { + n := map[string]string{} + for k, v := range m { + n[k] = v + } + return n +} + +func copyStringSlice(s []string) []string { + t := make([]string, len(s)) + copy(t, s) + return t +} + +// AddImageNames adds the specified names to the specified image. +func AddImageNames(store storage.Store, image *storage.Image, addNames []string) error { + names, err := ExpandNames(addNames) + if err != nil { + return err + } + err = store.SetNames(image.ID, append(image.Names, names...)) + if err != nil { + return errors.Wrapf(err, "error adding names (%v) to image %q", names, image.ID) + } + return nil +} + +// ExpandNames takes unqualified names, parses them as image names, and returns +// the fully expanded result, including a tag. Names which don't include a registry +// name will be marked for the most-preferred registry (i.e., the first one in our +// configuration). +func ExpandNames(names []string) ([]string, error) { + expanded := make([]string, 0, len(names)) + for _, n := range names { + name, err := reference.ParseNormalizedNamed(n) + if err != nil { + return nil, errors.Wrapf(err, "error parsing name %q", n) + } + name = reference.TagNameOnly(name) + tag := "" + digest := "" + if tagged, ok := name.(reference.NamedTagged); ok { + tag = ":" + tagged.Tag() + } + if digested, ok := name.(reference.Digested); ok { + digest = "@" + digested.Digest().String() + } + expanded = append(expanded, name.Name()+tag+digest) + } + return expanded, nil +} diff --git a/libpod/container_api.go b/libpod/container_api.go index 6106cfc12..5ebb7b091 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -1,7 +1,6 @@ package libpod import ( - "io" "io/ioutil" "os" "strconv" @@ -10,10 +9,8 @@ import ( "github.com/docker/docker/daemon/caps" "github.com/docker/docker/pkg/stringid" - ociv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/projectatomic/libpod/libpod/driver" - "github.com/projectatomic/libpod/libpod/image" "github.com/projectatomic/libpod/pkg/inspect" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" @@ -586,42 +583,6 @@ func (c *Container) Inspect(size bool) (*inspect.ContainerInspectData, error) { return c.getContainerInspectData(size, driverData) } -// Commit commits the changes between a container and its image, creating a new -// image -func (c *Container) Commit(pause bool, reference string, writer io.Writer, signingOptions image.SigningOptions, imageConfig ociv1.Image) (*image.Image, error) { - if !c.locked { - c.lock.Lock() - defer c.lock.Unlock() - - if err := c.syncContainer(); err != nil { - return nil, err - } - } - - if c.state.State == ContainerStateRunning && pause { - if err := c.runtime.ociRuntime.pauseContainer(c); err != nil { - return nil, errors.Wrapf(err, "error pausing container %q", c.ID()) - } - defer func() { - if err := c.runtime.ociRuntime.unpauseContainer(c); err != nil { - logrus.Errorf("error unpausing container %q: %v", c.ID(), err) - } - }() - } - - tempFile, err := ioutil.TempFile(c.runtime.config.TmpDir, "podman-commit") - if err != nil { - return nil, errors.Wrapf(err, "error creating temp file") - } - defer os.Remove(tempFile.Name()) - defer tempFile.Close() - - if err := c.export(tempFile.Name()); err != nil { - return nil, err - } - return c.runtime.imageRuntime.Import(tempFile.Name(), reference, writer, signingOptions, imageConfig) -} - // Wait blocks on a container to exit and returns its exit code func (c *Container) Wait() (int32, error) { if !c.valid { diff --git a/libpod/container_commit.go b/libpod/container_commit.go new file mode 100644 index 000000000..7ad393b6c --- /dev/null +++ b/libpod/container_commit.go @@ -0,0 +1,103 @@ +package libpod + +import ( + "strings" + + is "github.com/containers/image/storage" + "github.com/pkg/errors" + "github.com/projectatomic/libpod/libpod/buildah" + "github.com/projectatomic/libpod/libpod/image" + "github.com/sirupsen/logrus" +) + +// ContainerCommitOptions is a struct used to commit a container to an image +// It uses buildah's CommitOptions as a base. Long-term we might wish to +// add these to the buildah struct once buildah is more integrated with +//libpod +type ContainerCommitOptions struct { + buildah.CommitOptions + Pause bool + Author string + Message string + Changes []string +} + +// Commit commits the changes between a container and its image, creating a new +// image +func (c *Container) Commit(destImage string, options ContainerCommitOptions) (*image.Image, error) { + if !c.locked { + c.lock.Lock() + defer c.lock.Unlock() + + if err := c.syncContainer(); err != nil { + return nil, err + } + } + + if c.state.State == ContainerStateRunning && options.Pause { + if err := c.runtime.ociRuntime.pauseContainer(c); err != nil { + return nil, errors.Wrapf(err, "error pausing container %q", c.ID()) + } + defer func() { + if err := c.runtime.ociRuntime.unpauseContainer(c); err != nil { + logrus.Errorf("error unpausing container %q: %v", c.ID(), err) + } + }() + } + + sc := image.GetSystemContext(options.SignaturePolicyPath, "", false) + builderOptions := buildah.ImportOptions{ + Container: c.ID(), + SignaturePolicyPath: options.SignaturePolicyPath, + } + commitOptions := buildah.CommitOptions{ + SignaturePolicyPath: options.SignaturePolicyPath, + ReportWriter: options.ReportWriter, + SystemContext: sc, + } + importBuilder, err := buildah.ImportBuilder(c.runtime.store, builderOptions) + if err != nil { + return nil, err + } + + if options.Author != "" { + importBuilder.SetMaintainer(options.Author) + } + if options.Message != "" { + importBuilder.SetComment(options.Message) + } + + // Process user changes + for _, change := range options.Changes { + splitChange := strings.Split(change, "=") + switch strings.ToUpper(splitChange[0]) { + case "CMD": + importBuilder.SetCmd(splitChange[1:]) + case "ENTRYPOINT": + importBuilder.SetEntrypoint(splitChange[1:]) + case "ENV": + importBuilder.SetEnv(splitChange[1], splitChange[2]) + case "EXPOSE": + importBuilder.SetPort(splitChange[1]) + case "LABEL": + importBuilder.SetLabel(splitChange[1], splitChange[2]) + case "STOPSIGNAL": + // No Set StopSignal + case "USER": + importBuilder.SetUser(splitChange[1]) + case "VOLUME": + importBuilder.AddVolume(splitChange[1]) + case "WORKDIR": + importBuilder.SetWorkDir(splitChange[1]) + } + } + imageRef, err := is.Transport.ParseStoreReference(c.runtime.store, destImage) + if err != nil { + return nil, err + } + + if err = importBuilder.Commit(imageRef, commitOptions); err != nil { + return nil, err + } + return c.runtime.imageRuntime.NewFromLocal(imageRef.DockerReference().String()) +} diff --git a/libpod/image/image.go b/libpod/image/image.go index c79844f7f..009c51886 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -457,8 +457,14 @@ func (i *Image) MatchesID(id string) bool { // toStorageReference returns a *storageReference from an Image func (i *Image) toStorageReference() (types.ImageReference, error) { + var lookupName string if i.storeRef == nil { - storeRef, err := is.Transport.ParseStoreReference(i.imageruntime.store, i.ID()) + if i.image != nil { + lookupName = i.ID() + } else { + lookupName = i.InputName + } + storeRef, err := is.Transport.ParseStoreReference(i.imageruntime.store, lookupName) if err != nil { return nil, err } |