diff options
42 files changed, 623 insertions, 429 deletions
@@ -4,10 +4,8 @@ approvers: - giuseppe - jwhonce - mheon - - mrunalp - rhatdan - TomSweeneyRedHat - - umohnani8 - vrothberg reviewers: - baude @@ -15,8 +13,8 @@ reviewers: - giuseppe - jwhonce - mheon - - mrunalp - rhatdan - TomSweeneyRedHat - - umohnani8 - vrothberg + - ashley-cui + - QiWang19 diff --git a/cmd/podman/containers/logs.go b/cmd/podman/containers/logs.go index 21c00505f..acc2ab1aa 100644 --- a/cmd/podman/containers/logs.go +++ b/cmd/podman/containers/logs.go @@ -37,7 +37,7 @@ var ( case registry.IsRemote() && len(args) > 1: return errors.New(cmd.Name() + " does not support multiple containers when run remotely") case logsOptions.Latest && len(args) > 0: - return errors.New("no containers can be specified when using 'latest'") + return errors.New("--latest and containers cannot be used together") case !logsOptions.Latest && len(args) < 1: return errors.New("specify at least one container name or ID to log") } diff --git a/cmd/podman/containers/mount.go b/cmd/podman/containers/mount.go index 1c358f4a2..f2df5e99e 100644 --- a/cmd/podman/containers/mount.go +++ b/cmd/podman/containers/mount.go @@ -78,6 +78,9 @@ func mount(_ *cobra.Command, args []string) error { var ( errs utils.OutputErrors ) + if len(args) > 0 && mountOpts.Latest { + return errors.Errorf("--latest and containers cannot be used together") + } reports, err := registry.ContainerEngine().ContainerMount(registry.GetContext(), args, mountOpts) if err != nil { return err diff --git a/cmd/podman/containers/restart.go b/cmd/podman/containers/restart.go index d02097f0e..5f6f9c35c 100644 --- a/cmd/podman/containers/restart.go +++ b/cmd/podman/containers/restart.go @@ -80,6 +80,9 @@ func restart(cmd *cobra.Command, args []string) error { if len(args) < 1 && !restartOptions.Latest && !restartOptions.All { return errors.Wrapf(define.ErrInvalidArg, "you must provide at least one container name or ID") } + if len(args) > 0 && restartOptions.Latest { + return errors.Wrapf(define.ErrInvalidArg, "--latest and containers cannot be used together") + } if cmd.Flag("time").Changed { restartOptions.Timeout = &restartTimeout diff --git a/cmd/podman/containers/restore.go b/cmd/podman/containers/restore.go index 413bcbc87..c996144e3 100644 --- a/cmd/podman/containers/restore.go +++ b/cmd/podman/containers/restore.go @@ -80,7 +80,7 @@ func restore(_ *cobra.Command, args []string) error { } } if (restoreOptions.All || restoreOptions.Latest) && argLen > 0 { - return errors.Errorf("no arguments are needed with --all or --latest") + return errors.Errorf("--all or --latest and containers cannot be used together") } if argLen < 1 && !restoreOptions.All && !restoreOptions.Latest && restoreOptions.Import == "" { return errors.Errorf("you must provide at least one name or id") diff --git a/cmd/podman/containers/start.go b/cmd/podman/containers/start.go index ccbe80317..1e58498b6 100644 --- a/cmd/podman/containers/start.go +++ b/cmd/podman/containers/start.go @@ -74,6 +74,9 @@ func start(cmd *cobra.Command, args []string) error { if len(args) == 0 && !startOptions.Latest { return errors.New("start requires at least one argument") } + if len(args) > 0 && startOptions.Latest { + return errors.Errorf("--latest and containers cannot be used together") + } if len(args) > 1 && startOptions.Attach { return errors.Errorf("you cannot start and attach multiple containers at once") } diff --git a/cmd/podman/containers/wait.go b/cmd/podman/containers/wait.go index 9bb39e978..4bc3d20e2 100644 --- a/cmd/podman/containers/wait.go +++ b/cmd/podman/containers/wait.go @@ -78,7 +78,7 @@ func wait(cmd *cobra.Command, args []string) error { return errors.Errorf("%q requires a name, id, or the \"--latest\" flag", cmd.CommandPath()) } if waitOptions.Latest && len(args) > 0 { - return errors.New("--latest and containers are not allowed") + return errors.New("--latest and containers cannot be used together") } waitOptions.Condition, err = define.StringToContainerStatus(waitCondition) diff --git a/cmd/podman/images/build.go b/cmd/podman/images/build.go index ff5c6ec09..00777c48c 100644 --- a/cmd/podman/images/build.go +++ b/cmd/podman/images/build.go @@ -206,14 +206,9 @@ func build(cmd *cobra.Command, args []string) error { } } - ie, err := registry.NewImageEngine(cmd, args) - if err != nil { - return err - } - var logfile *os.File if cmd.Flag("logfile").Changed { - logfile, err = os.OpenFile(buildOpts.Logfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + logfile, err := os.OpenFile(buildOpts.Logfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) if err != nil { return errors.Errorf("error opening logfile %q: %v", buildOpts.Logfile, err) } @@ -225,7 +220,7 @@ func build(cmd *cobra.Command, args []string) error { return err } - _, err = ie.Build(registry.GetContext(), containerFiles, *apiBuildOpts) + _, err = registry.ImageEngine().Build(registry.GetContext(), containerFiles, *apiBuildOpts) return err } diff --git a/cmd/podman/inspect/inspect.go b/cmd/podman/inspect/inspect.go index 13c6544bb..f29527412 100644 --- a/cmd/podman/inspect/inspect.go +++ b/cmd/podman/inspect/inspect.go @@ -93,7 +93,7 @@ func (i *inspector) inspect(namesOrIDs []string) error { tmpType := i.options.Type if i.options.Latest { if len(namesOrIDs) > 0 { - return errors.New("latest and containers are not allowed") + return errors.New("--latest and containers cannot be used together") } tmpType = ContainerType // -l works with --type=all } diff --git a/cmd/podman/pods/inspect.go b/cmd/podman/pods/inspect.go index 8f5aa6062..bc20352b0 100644 --- a/cmd/podman/pods/inspect.go +++ b/cmd/podman/pods/inspect.go @@ -46,6 +46,9 @@ func inspect(cmd *cobra.Command, args []string) error { if len(args) < 1 && !inspectOptions.Latest { return errors.Errorf("you must provide the name or id of a running pod") } + if len(args) > 0 && inspectOptions.Latest { + return errors.Errorf("--latest and containers cannot be used together") + } if !inspectOptions.Latest { inspectOptions.NameOrID = args[0] diff --git a/cmd/podman/root.go b/cmd/podman/root.go index 60725b111..d079018a0 100644 --- a/cmd/podman/root.go +++ b/cmd/podman/root.go @@ -273,7 +273,6 @@ func rootFlags(cmd *cobra.Command, opts *entities.PodmanConfig) { pFlags.StringVar(&opts.RegistriesConf, "registries-conf", "", "Path to a registries.conf to use for image processing") pFlags.StringVar(&opts.Runroot, "runroot", "", "Path to the 'run directory' where all state information is stored") pFlags.StringVar(&opts.RuntimePath, "runtime", "", "Path to the OCI-compatible binary used to run containers, default is /usr/bin/runc") - pFlags.StringArrayVar(&opts.RuntimeFlags, "runtime-flag", []string{}, "add global flags for the container runtime") // -s is deprecated due to conflict with -s on subcommands pFlags.StringVar(&opts.StorageDriver, "storage-driver", "", "Select which storage driver is used to manage storage of images and containers (default is overlay)") pFlags.StringArrayVar(&opts.StorageOpts, "storage-opt", []string{}, "Used to pass an option to the storage driver") @@ -301,6 +300,7 @@ func rootFlags(cmd *cobra.Command, opts *entities.PodmanConfig) { // Only create these flags for ABI connections if !registry.IsRemote() { + pFlags.StringArrayVar(&opts.RuntimeFlags, "runtime-flag", []string{}, "add global flags for the container runtime") pFlags.BoolVar(&useSyslog, "syslog", false, "Output logging information to syslog as well as the console (default false)") } } diff --git a/cmd/podman/validate/args.go b/cmd/podman/validate/args.go index aacb41e69..ab6460e93 100644 --- a/cmd/podman/validate/args.go +++ b/cmd/podman/validate/args.go @@ -37,6 +37,9 @@ func IDOrLatestArgs(cmd *cobra.Command, args []string) error { if len(args) == 0 && !given { return fmt.Errorf("%q requires a name, id, or the \"--latest\" flag", cmd.CommandPath()) } + if len(args) > 0 && given { + return fmt.Errorf("--latest and containers cannot be used together") + } } return nil } @@ -83,7 +86,7 @@ func CheckAllLatestAndCIDFile(c *cobra.Command, args []string, ignoreArgLen bool if argLen > 0 { if specifiedLatest { - return errors.Errorf("no arguments are needed with --latest") + return errors.Errorf("--latest and containers cannot be used together") } else if cidfile && (specifiedLatest || specifiedCIDFile) { return errors.Errorf("no arguments are needed with --latest or --cidfile") } @@ -138,7 +141,7 @@ func CheckAllLatestAndPodIDFile(c *cobra.Command, args []string, ignoreArgLen bo if argLen > 0 { if specifiedLatest { - return errors.Errorf("no arguments are needed with --latest") + return errors.Errorf("--latest and pods cannot be used together") } else if withIDFile && (specifiedLatest || specifiedPodIDFile) { return errors.Errorf("no arguments are needed with --latest or --pod-id-file") } diff --git a/contrib/rootless-cni-infra/Containerfile b/contrib/rootless-cni-infra/Containerfile index c5d812a6e..5be30ccc9 100644 --- a/contrib/rootless-cni-infra/Containerfile +++ b/contrib/rootless-cni-infra/Containerfile @@ -33,3 +33,5 @@ COPY --from=dnsname /dnsname /opt/cni/bin COPY rootless-cni-infra /usr/local/bin ENV CNI_PATH=/opt/cni/bin CMD ["sleep", "infinity"] + +ENV ROOTLESS_CNI_INFRA_VERSION=1 diff --git a/contrib/rootless-cni-infra/README.md b/contrib/rootless-cni-infra/README.md index 937e057fb..5aa13374b 100644 --- a/contrib/rootless-cni-infra/README.md +++ b/contrib/rootless-cni-infra/README.md @@ -16,6 +16,8 @@ Podman then allocates a CNI netns in the infra container, by executing an equiva The allocated netns is deallocated when the container is being removed, by executing an equivalent of: `podman exec rootless-cni-infra rootless-cni-infra dealloc $CONTAINER_ID $NETWORK_NAME`. +The container images live on `quay.io/libpod/rootless-cni-infra`. The tags have the format `$version-$architecture`. Please make sure to increase the version number in the Containerfile (i.e., `ROOTLESS_CNI_INFRA_VERSION`) when applying changes to this directory. After committing the changes, upload the image(s) with the corresponding tag. + ## Directory layout * `/run/rootless-cni-infra/${CONTAINER_ID}/pid`: PID of the `sleep infinity` process that corresponds to the allocated netns diff --git a/contrib/rootless-cni-infra/rootless-cni-infra b/contrib/rootless-cni-infra/rootless-cni-infra index 5a574d2eb..f6622b23c 100755 --- a/contrib/rootless-cni-infra/rootless-cni-infra +++ b/contrib/rootless-cni-infra/rootless-cni-infra @@ -2,7 +2,6 @@ set -eu ARG0="$0" -VERSION="0.1.0" BASE="/run/rootless-cni-infra" # CLI subcommand: "alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME" @@ -126,7 +125,7 @@ cmd_entrypoint_help() { # CLI subcommand: "version" cmd_entrypoint_version() { - echo "{\"version\": \"${VERSION}\"}" + echo "{\"version\": \"${ROOTLESS_CNI_INFRA_VERSION}\"}" } # parse args diff --git a/docs/source/markdown/podman-remote.1.md b/docs/source/markdown/podman-remote.1.md index 3dcfae606..b621c846a 100644 --- a/docs/source/markdown/podman-remote.1.md +++ b/docs/source/markdown/podman-remote.1.md @@ -39,6 +39,11 @@ Path to ssh identity file. If the identity file has been encrypted, Podman promp If no identity file is provided and no user is given, Podman defaults to the user running the podman command. Podman prompts for the login password on the remote server. +Identity value resolution precedence: + - command line value + - environment variable `CONTAINER_SSHKEY`, if `CONTAINER_HOST` is found + - `containers.conf` + **--log-level**=*level* Log messages above specified level: debug, info, warn, error (default), fatal or panic @@ -47,6 +52,21 @@ Log messages above specified level: debug, info, warn, error (default), fatal or URL to access Podman service (default from `containers.conf`, rootless "unix://run/user/$UID/podman/podman.sock" or as root "unix://run/podman/podman.sock). + - `CONTAINER_HOST` is of the format `<schema>://[<user[:<password>]@]<host>[:<port>][<path>]` + +Details: + - `user` will default to either `root` or current running user + - `password` has no default + - `host` must be provided and is either the IP or name of the machine hosting the Podman service + - `port` defaults to 22 + - `path` defaults to either `/run/podman/podman.sock`, or `/run/user/<uid>/podman/podman.sock` if running rootless. + +URL value resolution precedence: + - command line value + - environment variable `CONTAINER_HOST` + - `containers.conf` + - `unix://run/podman/podman.sock` + **--version** Print the version @@ -124,3 +144,15 @@ the exit codes follow the `chroot` standard, see below: | [podman-unpause(1)](podman-unpause.1.md) | Unpause one or more containers. | | [podman-version(1)](podman-version.1.md) | Display the Podman version information. | | [podman-volume(1)](podman-volume.1.md) | Manage Volumes. | +## FILES + +**containers.conf** (`$HOME/.config/containers/containers.conf`) + +Podman has builtin defaults for command line options. These defaults can be overridden using the containers.conf configuration files. + +Users can modify defaults by creating the `$HOME/.config/containers/containers.conf` file. Podman merges its builtin defaults with the specified fields from this file, if it exists. Fields specified in the users file override the built-in defaults. + +Podman uses builtin defaults if no containers.conf file is found. + +## SEE ALSO +`containers.conf(5)` diff --git a/docs/source/markdown/podman.1.md b/docs/source/markdown/podman.1.md index 2dc6b13bf..555486562 100644 --- a/docs/source/markdown/podman.1.md +++ b/docs/source/markdown/podman.1.md @@ -67,6 +67,11 @@ Path to ssh identity file. If the identity file has been encrypted, podman promp If no identity file is provided and no user is given, podman defaults to the user running the podman command. Podman prompts for the login password on the remote server. +Identity value resolution precedence: + - command line value + - environment variable `CONTAINER_SSHKEY`, if `CONTAINER_HOST` is found + - `containers.conf` + **--log-level**=*level* Log messages above specified level: debug, info, warn, error (default), fatal or panic (default: "error") @@ -83,7 +88,22 @@ Path to the command binary to use for setting up a network. It is currently onl Access Podman service will be remote **--url**=*value* -URL to access Podman service (default from `containers.conf`, rootless "unix://run/user/$UID/podman/podman.sock" or as root "unix://run/podman/podman.sock). +URL to access Podman service (default from `containers.conf`, rootless `unix://run/user/$UID/podman/podman.sock` or as root `unix://run/podman/podman.sock`). + + - `CONTAINER_HOST` is of the format `<schema>://[<user[:<password>]@]<host>[:<port>][<path>]` + +Details: + - `user` will default to either `root` or current running user + - `password` has no default + - `host` must be provided and is either the IP or name of the machine hosting the Podman service + - `port` defaults to 22 + - `path` defaults to either `/run/podman/podman.sock`, or `/run/user/<uid>/podman/podman.sock` if running rootless. + +URL value resolution precedence: + - command line value + - environment variable `CONTAINER_HOST` + - `containers.conf` + - `unix://run/podman/podman.sock` **--root**=*value* @@ -13,7 +13,7 @@ require ( github.com/containers/buildah v1.16.1 github.com/containers/common v0.22.0 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.5.2 + github.com/containers/image/v5 v5.6.0 github.com/containers/psgo v1.5.1 github.com/containers/storage v1.23.5 github.com/coreos/go-systemd/v22 v22.1.0 diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 3bdf28e8c..dde7cafb1 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -635,7 +635,7 @@ func (c *Container) setupSystemd(mounts []spec.Mount, g generate.Generator) erro Destination: "/sys/fs/cgroup/systemd", Type: "bind", Source: "/sys/fs/cgroup/systemd", - Options: []string{"bind", "nodev", "nosuid", "rprivate"}, + Options: []string{"bind", "nodev", "noexec", "nosuid", "rprivate"}, } g.AddMount(systemdMnt) g.AddLinuxMaskedPaths("/sys/fs/cgroup/systemd/release_agent") diff --git a/libpod/rootless_cni_linux.go b/libpod/rootless_cni_linux.go index 76dbfdcae..31097dd16 100644 --- a/libpod/rootless_cni_linux.go +++ b/libpod/rootless_cni_linux.go @@ -13,6 +13,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containers/podman/v2/libpod/define" "github.com/containers/podman/v2/libpod/image" + "github.com/containers/podman/v2/pkg/env" "github.com/containers/podman/v2/pkg/util" "github.com/containers/storage/pkg/lockfile" "github.com/hashicorp/go-multierror" @@ -22,10 +23,9 @@ import ( "github.com/sirupsen/logrus" ) +// Built from ../contrib/rootless-cni-infra. var rootlessCNIInfraImage = map[string]string{ - // Built from ../contrib/rootless-cni-infra - // TODO: move to Podman's official quay - "amd64": "ghcr.io/akihirosuda/podman-rootless-cni-infra:gd34868a13-amd64", + "amd64": "quay.io/libpod/rootless-cni-infra@sha256:8aa681c4c08dee3ec5d46ff592fddd0259a35626717006d6b77ee786b1d02967", // 1-amd64 } const ( @@ -258,9 +258,23 @@ func startRootlessCNIInfraContainer(ctx context.Context, r *Runtime) (*Container Options: []string{"ro"}, } g.AddMount(etcCNINetD) - // FIXME: how to propagate ProcessArgs and Envs from Dockerfile? - g.SetProcessArgs([]string{"sleep", "infinity"}) - g.AddProcessEnv("CNI_PATH", "/opt/cni/bin") + + inspectData, err := newImage.Inspect(ctx) + if err != nil { + return nil, err + } + imageEnv, err := env.ParseSlice(inspectData.Config.Env) + if err != nil { + return nil, err + } + for k, v := range imageEnv { + g.AddProcessEnv(k, v) + } + if len(inspectData.Config.Cmd) == 0 { + return nil, errors.Errorf("rootless CNI infra image %q has no command specified", imageName) + } + g.SetProcessArgs(inspectData.Config.Cmd) + var options []CtrCreateOption options = append(options, WithRootFSFromImage(newImage.ID(), imageName, imageName)) options = append(options, WithCtrNamespace(rootlessCNIInfraContainerNamespace)) diff --git a/pkg/api/handlers/compat/events.go b/pkg/api/handlers/compat/events.go index 289bf4a2d..fbb33410f 100644 --- a/pkg/api/handlers/compat/events.go +++ b/pkg/api/handlers/compat/events.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "sync" "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/events" @@ -113,8 +112,13 @@ func GetEvents(w http.ResponseWriter, r *http.Request) { errorChannel <- runtime.Events(r.Context(), readOpts) }() - var coder *jsoniter.Encoder - var writeHeader sync.Once + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + coder := json.NewEncoder(w) + coder.SetEscapeHTML(true) for stream := true; stream; stream = query.Stream { select { @@ -124,18 +128,6 @@ func GetEvents(w http.ResponseWriter, r *http.Request) { return } case evt := <-eventChannel: - writeHeader.Do(func() { - // Use a sync.Once so that we write the header - // only once. - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } - coder = json.NewEncoder(w) - coder.SetEscapeHTML(true) - }) - if evt == nil { continue } diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 9601f5e18..fbaf8d10a 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -1,7 +1,7 @@ package compat import ( - "bytes" + "context" "encoding/base64" "encoding/json" "fmt" @@ -18,8 +18,10 @@ import ( "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/pkg/api/handlers" "github.com/containers/podman/v2/pkg/api/handlers/utils" + "github.com/containers/podman/v2/pkg/channel" "github.com/containers/storage/pkg/archive" "github.com/gorilla/schema" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -47,12 +49,25 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } } - anchorDir, err := extractTarFile(r) + contextDirectory, err := extractTarFile(r) if err != nil { utils.InternalServerError(w, err) return } - defer os.RemoveAll(anchorDir) + + defer func() { + if logrus.IsLevelEnabled(logrus.DebugLevel) { + if v, found := os.LookupEnv("PODMAN_RETAIN_BUILD_ARTIFACT"); found { + if keep, _ := strconv.ParseBool(v); keep { + return + } + } + } + err := os.RemoveAll(filepath.Dir(contextDirectory)) + if err != nil { + logrus.Warn(errors.Wrapf(err, "failed to remove build scratch directory %q", filepath.Dir(contextDirectory))) + } + }() query := struct { Dockerfile string `schema:"dockerfile"` @@ -67,10 +82,10 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { ForceRm bool `schema:"forcerm"` Memory int64 `schema:"memory"` MemSwap int64 `schema:"memswap"` - CpuShares uint64 `schema:"cpushares"` //nolint - CpuSetCpus string `schema:"cpusetcpus"` //nolint - CpuPeriod uint64 `schema:"cpuperiod"` //nolint - CpuQuota int64 `schema:"cpuquota"` //nolint + CpuShares uint64 `schema:"cpushares"` // nolint + CpuSetCpus string `schema:"cpusetcpus"` // nolint + CpuPeriod uint64 `schema:"cpuperiod"` // nolint + CpuQuota int64 `schema:"cpuquota"` // nolint BuildArgs string `schema:"buildargs"` ShmSize int `schema:"shmsize"` Squash bool `schema:"squash"` @@ -81,52 +96,32 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { Outputs string `schema:"outputs"` Registry string `schema:"registry"` }{ - Dockerfile: "Dockerfile", - Tag: []string{}, - ExtraHosts: "", - Remote: "", - Quiet: false, - NoCache: false, - CacheFrom: "", - Pull: false, - Rm: true, - ForceRm: false, - Memory: 0, - MemSwap: 0, - CpuShares: 0, - CpuSetCpus: "", - CpuPeriod: 0, - CpuQuota: 0, - BuildArgs: "", - ShmSize: 64 * 1024 * 1024, - Squash: false, - Labels: "", - NetworkMode: "", - Platform: "", - Target: "", - Outputs: "", - Registry: "docker.io", + Dockerfile: "Dockerfile", + Tag: []string{}, + Rm: true, + ShmSize: 64 * 1024 * 1024, + Registry: "docker.io", } + decoder := r.Context().Value("decoder").(*schema.Decoder) if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, err) return } - var ( - output string - additionalNames []string - ) + var output string if len(query.Tag) > 0 { output = query.Tag[0] } + if _, found := r.URL.Query()["target"]; found { + output = query.Target + } + + var additionalNames []string if len(query.Tag) > 1 { additionalNames = query.Tag[1:] } - if _, found := r.URL.Query()["target"]; found { - output = query.Target - } var buildArgs = map[string]string{} if _, found := r.URL.Query()["buildargs"]; found { if err := json.Unmarshal([]byte(query.BuildArgs), &buildArgs); err != nil { @@ -156,95 +151,136 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } } - // build events will be recorded here - var ( - buildEvents = []string{} - progress = bytes.Buffer{} - ) + // Channels all mux'ed in select{} below to follow API build protocol + stdout := channel.NewWriter(make(chan []byte, 1)) + defer stdout.Close() + + auxout := channel.NewWriter(make(chan []byte, 1)) + defer auxout.Close() + + stderr := channel.NewWriter(make(chan []byte, 1)) + defer stderr.Close() + + reporter := channel.NewWriter(make(chan []byte, 1)) + defer reporter.Close() buildOptions := imagebuildah.BuildOptions{ - ContextDirectory: filepath.Join(anchorDir, "build"), + ContextDirectory: contextDirectory, PullPolicy: pullPolicy, Registry: query.Registry, IgnoreUnrecognizedInstructions: true, Quiet: query.Quiet, Isolation: buildah.IsolationChroot, - Runtime: "", - RuntimeArgs: nil, - TransientMounts: nil, Compression: archive.Gzip, Args: buildArgs, Output: output, AdditionalTags: additionalNames, - Log: func(format string, args ...interface{}) { - buildEvents = append(buildEvents, fmt.Sprintf(format, args...)) - }, - In: nil, - Out: &progress, - Err: &progress, - SignaturePolicyPath: "", - ReportWriter: &progress, - OutputFormat: buildah.Dockerv2ImageManifest, - SystemContext: nil, - NamespaceOptions: nil, - ConfigureNetwork: 0, - CNIPluginPath: "", - CNIConfigDir: "", - IDMappingOptions: nil, - AddCapabilities: nil, - DropCapabilities: nil, + Out: stdout, + Err: auxout, + ReportWriter: reporter, + OutputFormat: buildah.Dockerv2ImageManifest, CommonBuildOpts: &buildah.CommonBuildOptions{ - AddHost: nil, - CgroupParent: "", - CPUPeriod: query.CpuPeriod, - CPUQuota: query.CpuQuota, - CPUShares: query.CpuShares, - CPUSetCPUs: query.CpuSetCpus, - CPUSetMems: "", - HTTPProxy: false, - Memory: query.Memory, - DNSSearch: nil, - DNSServers: nil, - DNSOptions: nil, - MemorySwap: query.MemSwap, - LabelOpts: nil, - SeccompProfilePath: "", - ApparmorProfile: "", - ShmSize: strconv.Itoa(query.ShmSize), - Ulimit: nil, - Volumes: nil, + CPUPeriod: query.CpuPeriod, + CPUQuota: query.CpuQuota, + CPUShares: query.CpuShares, + CPUSetCPUs: query.CpuSetCpus, + Memory: query.Memory, + MemorySwap: query.MemSwap, + ShmSize: strconv.Itoa(query.ShmSize), }, - DefaultMountsFilePath: "", - IIDFile: "", Squash: query.Squash, Labels: labels, - Annotations: nil, - OnBuild: nil, - Layers: false, NoCache: query.NoCache, RemoveIntermediateCtrs: query.Rm, ForceRmIntermediateCtrs: query.ForceRm, - BlobDirectory: "", Target: query.Target, - Devices: nil, } runtime := r.Context().Value("runtime").(*libpod.Runtime) - id, _, err := runtime.Build(r.Context(), buildOptions, query.Dockerfile) - if err != nil { - utils.InternalServerError(w, err) - return + runCtx, cancel := context.WithCancel(context.Background()) + var imageID string + go func() { + defer cancel() + imageID, _, err = runtime.Build(r.Context(), buildOptions, query.Dockerfile) + if err != nil { + stderr.Write([]byte(err.Error() + "\n")) + } + }() + + flush := func() { + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } } - // Find image ID that was built... - utils.WriteResponse(w, http.StatusOK, - struct { - Stream string `json:"stream"` - }{ - Stream: progress.String() + "\n" + - strings.Join(buildEvents, "\n") + - fmt.Sprintf("\nSuccessfully built %s\n", id), - }) + // Send headers and prime client for stream to come + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-Type", "application/json") + flush() + + var failed bool + + body := w.(io.Writer) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + if v, found := os.LookupEnv("PODMAN_RETAIN_BUILD_ARTIFACT"); found { + if keep, _ := strconv.ParseBool(v); keep { + t, _ := ioutil.TempFile("", "build_*_server") + defer t.Close() + body = io.MultiWriter(t, w) + } + } + } + + enc := json.NewEncoder(body) + enc.SetEscapeHTML(true) +loop: + for { + m := struct { + Stream string `json:"stream,omitempty"` + Error string `json:"error,omitempty"` + }{} + + select { + case e := <-stdout.Chan(): + m.Stream = string(e) + if err := enc.Encode(m); err != nil { + stderr.Write([]byte(err.Error())) + } + flush() + case e := <-auxout.Chan(): + m.Stream = string(e) + if err := enc.Encode(m); err != nil { + stderr.Write([]byte(err.Error())) + } + flush() + case e := <-reporter.Chan(): + m.Stream = string(e) + if err := enc.Encode(m); err != nil { + stderr.Write([]byte(err.Error())) + } + flush() + case e := <-stderr.Chan(): + failed = true + m.Error = string(e) + if err := enc.Encode(m); err != nil { + logrus.Warnf("Failed to json encode error %q", err.Error()) + } + flush() + case <-runCtx.Done(): + if !failed { + if utils.IsLibpodRequest(r) { + m.Stream = imageID + } else { + m.Stream = fmt.Sprintf("Successfully built %12.12s\n", imageID) + } + if err := enc.Encode(m); err != nil { + logrus.Warnf("Failed to json encode error %q", err.Error()) + } + flush() + } + break loop + } + } } func extractTarFile(r *http.Request) (string, error) { @@ -253,10 +289,9 @@ func extractTarFile(r *http.Request) (string, error) { if err != nil { return "", err } - buildDir := filepath.Join(anchorDir, "build") path := filepath.Join(anchorDir, "tarBall") - tarBall, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + tarBall, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return "", err } @@ -265,14 +300,17 @@ func extractTarFile(r *http.Request) (string, error) { // Content-Length not used as too many existing API clients didn't honor it _, err = io.Copy(tarBall, r.Body) r.Body.Close() - if err != nil { return "", fmt.Errorf("failed Request: Unable to copy tar file from request body %s", r.RequestURI) } - _, _ = tarBall.Seek(0, 0) - if err := archive.Untar(tarBall, buildDir, &archive.TarOptions{}); err != nil { + buildDir := filepath.Join(anchorDir, "build") + err = os.Mkdir(buildDir, 0700) + if err != nil { return "", err } - return anchorDir, nil + + _, _ = tarBall.Seek(0, 0) + err = archive.Untar(tarBall, buildDir, nil) + return buildDir, err } diff --git a/pkg/api/handlers/compat/ping.go b/pkg/api/handlers/compat/ping.go index eb7eed5b6..06150bb63 100644 --- a/pkg/api/handlers/compat/ping.go +++ b/pkg/api/handlers/compat/ping.go @@ -5,7 +5,6 @@ import ( "net/http" "github.com/containers/buildah" - "github.com/containers/podman/v2/pkg/api/handlers/utils" ) // Ping returns headers to client about the service @@ -14,13 +13,12 @@ import ( // Clients will use the Header availability to test which backend engine is in use. // Note: Additionally handler supports GET and HEAD methods func Ping(w http.ResponseWriter, r *http.Request) { - w.Header().Set("API-Version", utils.APIVersion[utils.CompatTree][utils.CurrentAPIVersion].String()) + // Note API-Version and Libpod-API-Version are set in handler_api.go w.Header().Set("BuildKit-Version", "") w.Header().Set("Docker-Experimental", "true") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Pragma", "no-cache") - w.Header().Set("Libpod-API-Version", utils.APIVersion[utils.LibpodTree][utils.CurrentAPIVersion].String()) w.Header().Set("Libpod-Buildha-Version", buildah.Version) w.WriteHeader(http.StatusOK) diff --git a/pkg/api/handlers/compat/version.go b/pkg/api/handlers/compat/version.go index e12c7cefa..92900b75d 100644 --- a/pkg/api/handlers/compat/version.go +++ b/pkg/api/handlers/compat/version.go @@ -30,6 +30,7 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "Failed to obtain system memory info")) return } + components := []docker.ComponentVersion{{ Name: "Podman Engine", Version: versionInfo.Version, @@ -46,6 +47,9 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { }, }} + apiVersion := utils.APIVersion[utils.CompatTree][utils.CurrentAPIVersion] + minVersion := utils.APIVersion[utils.CompatTree][utils.MinimalAPIVersion] + utils.WriteResponse(w, http.StatusOK, entities.ComponentVersion{ Version: docker.Version{ Platform: struct { @@ -53,7 +57,7 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { }{ Name: fmt.Sprintf("%s/%s/%s-%s", goRuntime.GOOS, goRuntime.GOARCH, infoData.Host.Distribution.Distribution, infoData.Host.Distribution.Version), }, - APIVersion: components[0].Details["APIVersion"], + APIVersion: fmt.Sprintf("%d.%d", apiVersion.Major, apiVersion.Minor), Arch: components[0].Details["Arch"], BuildTime: components[0].Details["BuildTime"], Components: components, @@ -61,7 +65,7 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { GitCommit: components[0].Details["GitCommit"], GoVersion: components[0].Details["GoVersion"], KernelVersion: components[0].Details["KernelVersion"], - MinAPIVersion: components[0].Details["MinAPIVersion"], + MinAPIVersion: fmt.Sprintf("%d.%d", minVersion.Major, minVersion.Minor), Os: components[0].Details["Os"], Version: components[0].Version, }}) diff --git a/pkg/api/handlers/utils/handler.go b/pkg/api/handlers/utils/handler.go index 62fdc05dd..517dccad0 100644 --- a/pkg/api/handlers/utils/handler.go +++ b/pkg/api/handlers/utils/handler.go @@ -43,8 +43,8 @@ var ( // clients to shop for the Version they wish to support APIVersion = map[VersionTree]map[VersionLevel]semver.Version{ LibpodTree: { - CurrentAPIVersion: semver.MustParse("1.0.0"), - MinimalAPIVersion: semver.MustParse("1.0.0"), + CurrentAPIVersion: semver.MustParse("2.0.0"), + MinimalAPIVersion: semver.MustParse("2.0.0"), }, CompatTree: { CurrentAPIVersion: semver.MustParse("1.40.0"), diff --git a/pkg/api/server/handler_api.go b/pkg/api/server/handler_api.go index e47b66bb4..f2ce0301b 100644 --- a/pkg/api/server/handler_api.go +++ b/pkg/api/server/handler_api.go @@ -40,6 +40,10 @@ func (s *APIServer) APIHandler(h http.HandlerFunc) http.HandlerFunc { c = context.WithValue(c, "idletracker", s.idleTracker) //nolint r = r.WithContext(c) + v := utils.APIVersion[utils.CompatTree][utils.CurrentAPIVersion] + w.Header().Set("API-Version", fmt.Sprintf("%d.%d", v.Major, v.Minor)) + w.Header().Set("Libpod-API-Version", utils.APIVersion[utils.LibpodTree][utils.CurrentAPIVersion].String()) + h(w, r) } fn(w, r) diff --git a/pkg/api/server/register_archive.go b/pkg/api/server/register_archive.go index 5931c2fc9..b2d2543c4 100644 --- a/pkg/api/server/register_archive.go +++ b/pkg/api/server/register_archive.go @@ -9,7 +9,7 @@ import ( ) func (s *APIServer) registerAchiveHandlers(r *mux.Router) error { - // swagger:operation POST /containers/{name}/archive compat putArchive + // swagger:operation PUT /containers/{name}/archive compat putArchive // --- // summary: Put files into a container // description: Put a tar archive of files into a container @@ -84,9 +84,9 @@ func (s *APIServer) registerAchiveHandlers(r *mux.Router) error { // $ref: "#/responses/NoSuchContainer" // 500: // $ref: "#/responses/InternalError" - r.HandleFunc(VersionedPath("/containers/{name}/archive"), s.APIHandler(compat.Archive)).Methods(http.MethodGet, http.MethodPost) + r.HandleFunc(VersionedPath("/containers/{name}/archive"), s.APIHandler(compat.Archive)).Methods(http.MethodGet, http.MethodPut, http.MethodHead) // Added non version path to URI to support docker non versioned paths - r.HandleFunc("/containers/{name}/archive", s.APIHandler(compat.Archive)).Methods(http.MethodGet, http.MethodPost) + r.HandleFunc("/containers/{name}/archive", s.APIHandler(compat.Archive)).Methods(http.MethodGet, http.MethodPut, http.MethodHead) /* Libpod diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go new file mode 100644 index 000000000..9082670a7 --- /dev/null +++ b/pkg/bindings/images/build.go @@ -0,0 +1,227 @@ +package images + +import ( + "archive/tar" + "context" + "encoding/json" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/containers/buildah" + "github.com/containers/podman/v2/pkg/bindings" + "github.com/containers/podman/v2/pkg/domain/entities" + "github.com/docker/go-units" + "github.com/hashicorp/go-multierror" + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// Build creates an image using a containerfile reference +func Build(ctx context.Context, containerFiles []string, options entities.BuildOptions) (*entities.BuildReport, error) { + params := url.Values{} + + if t := options.Output; len(t) > 0 { + params.Set("t", t) + } + for _, tag := range options.AdditionalTags { + params.Add("t", tag) + } + if options.Quiet { + params.Set("q", "1") + } + if options.NoCache { + params.Set("nocache", "1") + } + // TODO cachefrom + if options.PullPolicy == buildah.PullAlways { + params.Set("pull", "1") + } + if options.RemoveIntermediateCtrs { + params.Set("rm", "1") + } + if options.ForceRmIntermediateCtrs { + params.Set("forcerm", "1") + } + if mem := options.CommonBuildOpts.Memory; mem > 0 { + params.Set("memory", strconv.Itoa(int(mem))) + } + if memSwap := options.CommonBuildOpts.MemorySwap; memSwap > 0 { + params.Set("memswap", strconv.Itoa(int(memSwap))) + } + if cpuShares := options.CommonBuildOpts.CPUShares; cpuShares > 0 { + params.Set("cpushares", strconv.Itoa(int(cpuShares))) + } + if cpuSetCpus := options.CommonBuildOpts.CPUSetCPUs; len(cpuSetCpus) > 0 { + params.Set("cpusetcpues", cpuSetCpus) + } + if cpuPeriod := options.CommonBuildOpts.CPUPeriod; cpuPeriod > 0 { + params.Set("cpuperiod", strconv.Itoa(int(cpuPeriod))) + } + if cpuQuota := options.CommonBuildOpts.CPUQuota; cpuQuota > 0 { + params.Set("cpuquota", strconv.Itoa(int(cpuQuota))) + } + if buildArgs := options.Args; len(buildArgs) > 0 { + bArgs, err := jsoniter.MarshalToString(buildArgs) + if err != nil { + return nil, err + } + params.Set("buildargs", bArgs) + } + if shmSize := options.CommonBuildOpts.ShmSize; len(shmSize) > 0 { + shmBytes, err := units.RAMInBytes(shmSize) + if err != nil { + return nil, err + } + params.Set("shmsize", strconv.Itoa(int(shmBytes))) + } + if options.Squash { + params.Set("squash", "1") + } + if labels := options.Labels; len(labels) > 0 { + l, err := jsoniter.MarshalToString(labels) + if err != nil { + return nil, err + } + params.Set("labels", l) + } + + stdout := io.Writer(os.Stdout) + if options.Out != nil { + stdout = options.Out + } + + // TODO network? + + var platform string + if OS := options.OS; len(OS) > 0 { + platform += OS + } + if arch := options.Architecture; len(arch) > 0 { + platform += "/" + arch + } + if len(platform) > 0 { + params.Set("platform", platform) + } + + entries := make([]string, len(containerFiles)) + copy(entries, containerFiles) + entries = append(entries, options.ContextDirectory) + tarfile, err := nTar(entries...) + if err != nil { + return nil, err + } + defer tarfile.Close() + params.Set("dockerfile", filepath.Base(containerFiles[0])) + + conn, err := bindings.GetClient(ctx) + if err != nil { + return nil, err + } + response, err := conn.DoRequest(tarfile, http.MethodPost, "/build", params, nil) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if !response.IsSuccess() { + return nil, response.Process(err) + } + + body := response.Body.(io.Reader) + if logrus.IsLevelEnabled(logrus.DebugLevel) { + if v, found := os.LookupEnv("PODMAN_RETAIN_BUILD_ARTIFACT"); found { + if keep, _ := strconv.ParseBool(v); keep { + t, _ := ioutil.TempFile("", "build_*_client") + defer t.Close() + body = io.TeeReader(response.Body, t) + } + } + } + + dec := json.NewDecoder(body) + re := regexp.MustCompile(`[0-9a-f]{12}`) + + var id string + for { + var s struct { + Stream string `json:"stream,omitempty"` + Error string `json:"error,omitempty"` + } + if err := dec.Decode(&s); err != nil { + if errors.Is(err, io.EOF) { + return &entities.BuildReport{ID: id}, nil + } + s.Error = err.Error() + "\n" + } + + switch { + case s.Stream != "": + stdout.Write([]byte(s.Stream)) + if re.Match([]byte(s.Stream)) { + id = s.Stream + } + case s.Error != "": + return nil, errors.New(s.Error) + default: + return &entities.BuildReport{ID: id}, errors.New("failed to parse build results stream, unexpected input") + } + } +} + +func nTar(sources ...string) (io.ReadCloser, error) { + if len(sources) == 0 { + return nil, errors.New("No source(s) provided for build") + } + + pr, pw := io.Pipe() + tw := tar.NewWriter(pw) + + var merr error + go func() { + defer pw.Close() + defer tw.Close() + + for _, src := range sources { + s := src + err := filepath.Walk(s, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() || path == s { + return nil + } + + f, lerr := os.Open(path) + if lerr != nil { + return lerr + } + + name := strings.TrimPrefix(path, s+string(filepath.Separator)) + hdr, lerr := tar.FileInfoHeader(info, name) + if lerr != nil { + f.Close() + return lerr + } + hdr.Name = name + if lerr := tw.WriteHeader(hdr); lerr != nil { + f.Close() + return lerr + } + + _, cerr := io.Copy(tw, f) + f.Close() + return cerr + }) + merr = multierror.Append(merr, err) + } + }() + return pr, merr +} diff --git a/pkg/bindings/images/images.go b/pkg/bindings/images/images.go index a80c94025..05ab25d5b 100644 --- a/pkg/bindings/images/images.go +++ b/pkg/bindings/images/images.go @@ -1,7 +1,6 @@ package images import ( - "bytes" "context" "fmt" "io" @@ -9,14 +8,11 @@ import ( "net/url" "strconv" - "github.com/containers/buildah" "github.com/containers/image/v5/types" "github.com/containers/podman/v2/pkg/api/handlers" "github.com/containers/podman/v2/pkg/auth" "github.com/containers/podman/v2/pkg/bindings" "github.com/containers/podman/v2/pkg/domain/entities" - "github.com/docker/go-units" - jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" ) @@ -242,112 +238,6 @@ func Untag(ctx context.Context, nameOrID, tag, repo string) error { return response.Process(nil) } -// Build creates an image using a containerfile reference -func Build(ctx context.Context, containerFiles []string, options entities.BuildOptions, tarfile io.Reader) (*entities.BuildReport, error) { - var ( - platform string - report entities.BuildReport - ) - conn, err := bindings.GetClient(ctx) - if err != nil { - return nil, err - } - params := url.Values{} - params.Set("dockerfile", containerFiles[0]) - if t := options.Output; len(t) > 0 { - params.Set("t", t) - } - for _, tag := range options.AdditionalTags { - params.Add("t", tag) - } - // TODO Remote, Quiet - if options.NoCache { - params.Set("nocache", "1") - } - // TODO cachefrom - if options.PullPolicy == buildah.PullAlways { - params.Set("pull", "1") - } - if options.RemoveIntermediateCtrs { - params.Set("rm", "1") - } - if options.ForceRmIntermediateCtrs { - params.Set("forcerm", "1") - } - if mem := options.CommonBuildOpts.Memory; mem > 0 { - params.Set("memory", strconv.Itoa(int(mem))) - } - if memSwap := options.CommonBuildOpts.MemorySwap; memSwap > 0 { - params.Set("memswap", strconv.Itoa(int(memSwap))) - } - if cpuShares := options.CommonBuildOpts.CPUShares; cpuShares > 0 { - params.Set("cpushares", strconv.Itoa(int(cpuShares))) - } - if cpuSetCpus := options.CommonBuildOpts.CPUSetCPUs; len(cpuSetCpus) > 0 { - params.Set("cpusetcpues", cpuSetCpus) - } - if cpuPeriod := options.CommonBuildOpts.CPUPeriod; cpuPeriod > 0 { - params.Set("cpuperiod", strconv.Itoa(int(cpuPeriod))) - } - if cpuQuota := options.CommonBuildOpts.CPUQuota; cpuQuota > 0 { - params.Set("cpuquota", strconv.Itoa(int(cpuQuota))) - } - if buildArgs := options.Args; len(buildArgs) > 0 { - bArgs, err := jsoniter.MarshalToString(buildArgs) - if err != nil { - return nil, err - } - params.Set("buildargs", bArgs) - } - if shmSize := options.CommonBuildOpts.ShmSize; len(shmSize) > 0 { - shmBytes, err := units.RAMInBytes(shmSize) - if err != nil { - return nil, err - } - params.Set("shmsize", strconv.Itoa(int(shmBytes))) - } - if options.Squash { - params.Set("squash", "1") - } - if labels := options.Labels; len(labels) > 0 { - l, err := jsoniter.MarshalToString(labels) - if err != nil { - return nil, err - } - params.Set("labels", l) - } - - // TODO network? - if OS := options.OS; len(OS) > 0 { - platform += OS - } - if arch := options.Architecture; len(arch) > 0 { - platform += "/" + arch - } - if len(platform) > 0 { - params.Set("platform", platform) - } - // TODO outputs? - - response, err := conn.DoRequest(tarfile, http.MethodPost, "/build", params, nil) - if err != nil { - return nil, err - } - var streamReponse []byte - bb := bytes.NewBuffer(streamReponse) - if _, err = io.Copy(bb, response.Body); err != nil { - return nil, err - } - var s struct { - Stream string `json:"stream"` - } - if err := jsoniter.UnmarshalFromString(bb.String(), &s); err != nil { - return nil, err - } - fmt.Print(s.Stream) - return &report, nil -} - // Imports adds the given image to the local image store. This can be done by file and the given reader // or via the url parameter. Additional metadata can be associated with the image by using the changes and // message parameters. The image can also be tagged given a reference. One of url OR r must be provided. diff --git a/pkg/channel/doc.go b/pkg/channel/doc.go new file mode 100644 index 000000000..656fbddaa --- /dev/null +++ b/pkg/channel/doc.go @@ -0,0 +1,17 @@ +/* +Package channel provides helper structs/methods/funcs for working with channels + +Proxy from an io.Writer to a channel: + + w := channel.NewWriter(make(chan []byte, 10)) + go func() { + w.Write([]byte("Hello, World")) + }() + + fmt.Println(string(<-w.Chan())) + w.Close() + +Use of the constructor is required to initialize the channel. +Provide a channel of sufficient size to handle messages from writer(s). +*/ +package channel diff --git a/pkg/channel/writer.go b/pkg/channel/writer.go new file mode 100644 index 000000000..dbb38e416 --- /dev/null +++ b/pkg/channel/writer.go @@ -0,0 +1,53 @@ +package channel + +import ( + "io" + "sync" + + "github.com/pkg/errors" +) + +// WriteCloser is an io.WriteCloser that that proxies Write() calls to a channel +// The []byte buffer of the Write() is queued on the channel as one message. +type WriteCloser interface { + io.WriteCloser + Chan() <-chan []byte +} + +type writeCloser struct { + ch chan []byte + mux sync.Mutex +} + +// NewWriter initializes a new channel writer +func NewWriter(c chan []byte) WriteCloser { + return &writeCloser{ + ch: c, + } +} + +// Chan returns the R/O channel behind WriteCloser +func (w *writeCloser) Chan() <-chan []byte { + return w.ch +} + +// Write method for WriteCloser +func (w *writeCloser) Write(b []byte) (int, error) { + if w == nil || w.ch == nil { + return 0, errors.New("use channel.NewWriter() to initialize a WriteCloser") + } + + w.mux.Lock() + buf := make([]byte, len(b)) + copy(buf, b) + w.ch <- buf + w.mux.Unlock() + + return len(b), nil +} + +// Close method for WriteCloser +func (w *writeCloser) Close() error { + close(w.ch) + return nil +} diff --git a/pkg/channelwriter/channelwriter.go b/pkg/channelwriter/channelwriter.go deleted file mode 100644 index d51400eb3..000000000 --- a/pkg/channelwriter/channelwriter.go +++ /dev/null @@ -1,34 +0,0 @@ -package channelwriter - -import "github.com/pkg/errors" - -// Writer is an io.writer-like object that "writes" to a channel -// instead of a buffer or file, etc. It is handy for varlink endpoints when -// needing to handle endpoints that do logging "real-time" -type Writer struct { - ByteChannel chan []byte -} - -// NewChannelWriter creates a new channel writer and adds a -// byte slice channel into it. -func NewChannelWriter() *Writer { - byteChannel := make(chan []byte) - return &Writer{ - ByteChannel: byteChannel, - } -} - -// Write method for Writer -func (c *Writer) Write(w []byte) (int, error) { - if c.ByteChannel == nil { - return 0, errors.New("channel writer channel cannot be nil") - } - c.ByteChannel <- w - return len(w), nil -} - -// Close method for Writer -func (c *Writer) Close() error { - close(c.ByteChannel) - return nil -} diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 23aef9573..cc3ec37fb 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -535,6 +535,7 @@ func (ir *ImageEngine) Config(_ context.Context) (*config.Config, error) { } func (ir *ImageEngine) Build(ctx context.Context, containerFiles []string, opts entities.BuildOptions) (*entities.BuildReport, error) { + id, _, err := ir.Libpod.Build(ctx, opts.BuildOptions, containerFiles...) if err != nil { return nil, err diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index 185cc2f9a..50b8342a3 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -1,13 +1,9 @@ package tunnel import ( - "archive/tar" - "bytes" "context" - "io" "io/ioutil" "os" - "path/filepath" "strings" "time" @@ -18,9 +14,7 @@ import ( "github.com/containers/podman/v2/pkg/domain/entities" "github.com/containers/podman/v2/pkg/domain/utils" utils2 "github.com/containers/podman/v2/utils" - "github.com/containers/storage/pkg/archive" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) func (ir *ImageEngine) Exists(_ context.Context, nameOrID string) (*entities.BoolReport, error) { @@ -311,28 +305,8 @@ func (ir *ImageEngine) Config(_ context.Context) (*config.Config, error) { return config.Default() } -func (ir *ImageEngine) Build(ctx context.Context, containerFiles []string, opts entities.BuildOptions) (*entities.BuildReport, error) { - var tarReader io.Reader - tarfile, err := archive.Tar(opts.ContextDirectory, 0) - if err != nil { - return nil, err - } - tarReader = tarfile - cwd, err := os.Getwd() - if err != nil { - return nil, err - } - if cwd != opts.ContextDirectory { - fn := func(h *tar.Header, r io.Reader) (data []byte, update bool, skip bool, err error) { - h.Name = filepath.Join(filepath.Base(opts.ContextDirectory), h.Name) - return nil, false, false, nil - } - tarReader, err = transformArchive(tarfile, false, fn) - if err != nil { - return nil, err - } - } - return images.Build(ir.ClientCxt, containerFiles, opts, tarReader) +func (ir *ImageEngine) Build(_ context.Context, containerFiles []string, opts entities.BuildOptions) (*entities.BuildReport, error) { + return images.Build(ir.ClientCxt, containerFiles, opts) } func (ir *ImageEngine) Tree(ctx context.Context, nameOrID string, opts entities.ImageTreeOptions) (*entities.ImageTreeReport, error) { @@ -346,65 +320,3 @@ func (ir *ImageEngine) Shutdown(_ context.Context) { func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entities.SignOptions) (*entities.SignReport, error) { return nil, errors.New("not implemented yet") } - -// Sourced from openshift image builder - -// TransformFileFunc is given a chance to transform an arbitrary input file. -type TransformFileFunc func(h *tar.Header, r io.Reader) (data []byte, update bool, skip bool, err error) - -// filterArchive transforms the provided input archive to a new archive, -// giving the fn a chance to transform arbitrary files. -func filterArchive(r io.Reader, w io.Writer, fn TransformFileFunc) error { - tr := tar.NewReader(r) - tw := tar.NewWriter(w) - - var body io.Reader = tr - - for { - h, err := tr.Next() - if err == io.EOF { - return tw.Close() - } - if err != nil { - return err - } - - name := h.Name - data, ok, skip, err := fn(h, tr) - logrus.Debugf("Transform %q -> %q: data=%t ok=%t skip=%t err=%v", name, h.Name, data != nil, ok, skip, err) - if err != nil { - return err - } - if skip { - continue - } - if ok { - h.Size = int64(len(data)) - body = bytes.NewBuffer(data) - } - if err := tw.WriteHeader(h); err != nil { - return err - } - if _, err := io.Copy(tw, body); err != nil { - return err - } - } -} - -func transformArchive(r io.Reader, compressed bool, fn TransformFileFunc) (io.Reader, error) { - var cwe error - pr, pw := io.Pipe() - go func() { - if compressed { - in, err := archive.DecompressStream(r) - if err != nil { - cwe = pw.CloseWithError(err) - return - } - r = in - } - err := filterArchive(r, pw, fn) - cwe = pw.CloseWithError(err) - }() - return pr, cwe -} diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index 1dcfa2213..4bcf70b0d 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -23,7 +23,7 @@ import ( "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/define" "github.com/containers/podman/v2/libpod/image" - "github.com/containers/podman/v2/pkg/channelwriter" + "github.com/containers/podman/v2/pkg/channel" "github.com/containers/podman/v2/pkg/util" iopodman "github.com/containers/podman/v2/pkg/varlink" "github.com/containers/podman/v2/utils" @@ -570,7 +570,7 @@ func (i *VarlinkAPI) Commit(call iopodman.VarlinkCall, name, imageName string, c log []string mimeType string ) - output := channelwriter.NewChannelWriter() + output := channel.NewWriter(make(chan []byte)) channelClose := func() { if err := output.Close(); err != nil { logrus.Errorf("failed to close channel writer: %q", err) @@ -704,7 +704,7 @@ func (i *VarlinkAPI) PullImage(call iopodman.VarlinkCall, name string, creds iop if call.WantsMore() { call.Continues = true } - output := channelwriter.NewChannelWriter() + output := channel.NewWriter(make(chan []byte)) channelClose := func() { if err := output.Close(); err != nil { logrus.Errorf("failed to close channel writer: %q", err) diff --git a/pkg/varlinkapi/util.go b/pkg/varlinkapi/util.go index 748d8f9cc..7f96965f0 100644 --- a/pkg/varlinkapi/util.go +++ b/pkg/varlinkapi/util.go @@ -11,7 +11,7 @@ import ( "github.com/containers/buildah" "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/define" - "github.com/containers/podman/v2/pkg/channelwriter" + "github.com/containers/podman/v2/pkg/channel" iopodman "github.com/containers/podman/v2/pkg/varlink" "github.com/containers/storage/pkg/archive" ) @@ -201,7 +201,7 @@ func makePsOpts(inOpts iopodman.PsOpts) PsOptions { // more. it is capable of sending updates as the output writer gets them or append them // all to a log. the chan error is the error from the libpod call so we can honor // and error event in that case. -func forwardOutput(log []string, c chan error, wantsMore bool, output *channelwriter.Writer, reply func(br iopodman.MoreResponse) error) ([]string, error) { +func forwardOutput(log []string, c chan error, wantsMore bool, output channel.WriteCloser, reply func(br iopodman.MoreResponse) error) ([]string, error) { done := false for { select { @@ -214,7 +214,7 @@ func forwardOutput(log []string, c chan error, wantsMore bool, output *channelwr done = true // if no error is found, we pull what we can from the log writer and // append it to log string slice - case line := <-output.ByteChannel: + case line := <-output.Chan(): log = append(log, string(line)) // If the end point is being used in more mode, send what we have if wantsMore { diff --git a/test/apiv2/01-basic.at b/test/apiv2/01-basic.at index 96b6aef7c..541d8cbf1 100644 --- a/test/apiv2/01-basic.at +++ b/test/apiv2/01-basic.at @@ -18,11 +18,11 @@ t HEAD libpod/_ping 200 for i in /version version; do t GET $i 200 \ .Components[0].Name="Podman Engine" \ - .Components[0].Details.APIVersion=1.0.0 \ - .Components[0].Details.MinAPIVersion=1.0.0 \ + .Components[0].Details.APIVersion=2.0.0 \ + .Components[0].Details.MinAPIVersion=2.0.0 \ .Components[0].Details.Os=linux \ - .ApiVersion=1.0.0 \ - .MinAPIVersion=1.0.0 \ + .ApiVersion=1.40 \ + .MinAPIVersion=1.24 \ .Os=linux done diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index b6bbae15b..2ce3f9760 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -432,7 +432,7 @@ func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers Expect(err).To(BeNil()) session := p.PodmanNoCache([]string{"build", "--layers=" + layers, "-t", imageName, "--file", dockerfilePath, p.TempDir}) session.Wait(120) - Expect(session.ExitCode()).To(Equal(0)) + Expect(session).Should(Exit(0), fmt.Sprintf("BuildImage session output: %q", session.OutputToString())) } // PodmanPID execs podman and returns its PID diff --git a/test/e2e/images_test.go b/test/e2e/images_test.go index b22964dc1..a615a9f99 100644 --- a/test/e2e/images_test.go +++ b/test/e2e/images_test.go @@ -185,6 +185,7 @@ RUN apk update && apk add strace result.WaitWithDefaultTimeout() Expect(result).Should(Exit(0)) Expect(len(result.OutputToStringArray()) >= 1).To(BeTrue()) + }) It("podman images workingdir from image", func() { @@ -226,7 +227,7 @@ WORKDIR /test result := podmanTest.PodmanNoCache([]string{"image", "list", "-q", "-f", "after=docker.io/library/alpine:latest"}) result.WaitWithDefaultTimeout() Expect(result).Should(Exit(0)) - Expect(len(result.OutputToStringArray())).To(Equal(0)) + Expect(result.OutputToStringArray()).Should(HaveLen(0), "list filter output: %q", result.OutputToString()) }) It("podman images filter dangling", func() { @@ -236,8 +237,8 @@ WORKDIR /test podmanTest.BuildImage(dockerfile, "foobar.com/before:latest", "false") result := podmanTest.Podman([]string{"images", "-q", "-f", "dangling=true"}) result.WaitWithDefaultTimeout() - Expect(result).Should(Exit(0)) - Expect(len(result.OutputToStringArray())).To(Equal(0)) + Expect(result).Should(Exit(0), "dangling image output: %q", result.OutputToString()) + Expect(result.OutputToStringArray()).Should(HaveLen(0), "dangling image output: %q", result.OutputToString()) }) It("podman check for image with sha256: prefix", func() { diff --git a/test/system/015-help.bats b/test/system/015-help.bats index 4a3781012..651fdcd09 100644 --- a/test/system/015-help.bats +++ b/test/system/015-help.bats @@ -86,6 +86,20 @@ function check_help() { found[takes_no_args]=1 fi + # If command lists "-l, --latest" in help output, combine -l with arg. + # This should be disallowed with a clear message. + if expr "$full_help" : ".*-l, --latest" >/dev/null; then + local nope="exec list port ps top" # these can't be tested + if is_rootless; then + nope="$nope mount restore" # these don't work rootless + fi + if ! grep -wq "$cmd" <<<$nope; then + run_podman 125 "$@" $cmd -l nonexistent-container + is "$output" "Error: .*--latest and \(containers\|pods\|arguments\) cannot be used together" \ + "'$command_string' with both -l and container" + fi + fi + # If usage has required arguments, try running without them. # The expression here is 'first capital letter is not in [BRACKETS]'. # It is intended to handle 'podman foo [flags] ARG' but not ' [ARG]'. diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 66f6610ea..e3a139b4f 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -1,4 +1,5 @@ #!/usr/bin/env bats -*- bats -*- +# shellcheck disable=SC2096 # # Tests for podman build # @@ -6,8 +7,6 @@ load helpers @test "podman build - basic test" { - skip_if_remote "FIXME: pending #7136" - rand_filename=$(random_string 20) rand_content=$(random_string 50) @@ -31,7 +30,7 @@ EOF } @test "podman build - global runtime flags test" { - skip_if_remote "FIXME: pending #7136" + skip_if_remote "--runtime-flag flag not supported for remote" rand_content=$(random_string 50) @@ -49,11 +48,6 @@ EOF # Regression from v1.5.0. This test passes fine in v1.5.0, fails in 1.6 @test "podman build - cache (#3920)" { - skip_if_remote "FIXME: pending #7136, runtime flag is not passing over remote" - if is_remote && is_rootless; then - skip "unreliable with podman-remote and rootless; #2972" - fi - # Make an empty test directory, with a subdirectory used for tar tmpdir=$PODMAN_TMPDIR/build-test mkdir -p $tmpdir/subtest || die "Could not mkdir $tmpdir/subtest" @@ -97,8 +91,6 @@ EOF } @test "podman build - URLs" { - skip_if_remote "FIXME: pending #7137" - tmpdir=$PODMAN_TMPDIR/build-test mkdir -p $tmpdir @@ -118,8 +110,6 @@ EOF @test "podman build - workdir, cmd, env, label" { - skip_if_remote "FIXME: pending #7137" - tmpdir=$PODMAN_TMPDIR/build-test mkdir -p $tmpdir @@ -194,8 +184,15 @@ EOF build_test is "${lines[0]}" "$workdir" "container default command: pwd" is "${lines[1]}" "$s_echo" "container default command: output from echo" + is "${lines[2]}" "$s_env1" "container default command: env1" - is "${lines[3]}" "$s_env2" "container default command: env2" + + if is_remote; then + is "${lines[3]}" "this-should-be-overridden-by-env-host" "podman-remote does not send local environment" + else + is "${lines[3]}" "$s_env2" "container default command: env2" + fi + is "${lines[4]}" "$s_env3" "container default command: env3 (from envfile)" is "${lines[5]}" "$s_env4" "container default command: env4 (from cmdline)" @@ -206,7 +203,12 @@ EOF printenv http_proxy https_proxy ftp_proxy is "${lines[0]}" "http-proxy-in-env-file" "env-file overrides env" is "${lines[1]}" "https-proxy-in-env-file" "env-file sets proxy var" - is "${lines[2]}" "ftp-proxy-from-env" "ftp-proxy is passed through" + + if is_remote; then + is "${lines[2]}" "ftp-proxy-in-image" "podman-remote does not send local environment" + else + is "${lines[2]}" "ftp-proxy-from-env" "ftp-proxy is passed through" + fi # test that workdir is set for command-line commands also run_podman run --rm build_test pwd @@ -271,8 +273,6 @@ Labels.$label_name | $label_value } @test "podman build - stdin test" { - skip_if_remote "FIXME: pending #7136" - # Random workdir, and random string to verify build output workdir=/$(random_string 10) random_echo=$(random_string 15) diff --git a/vendor/modules.txt b/vendor/modules.txt index ffd90f5a5..9959c9efb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -99,7 +99,7 @@ github.com/containers/common/pkg/sysinfo github.com/containers/common/version # github.com/containers/conmon v2.0.20+incompatible github.com/containers/conmon/runner/config -# github.com/containers/image/v5 v5.5.2 => github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0 +# github.com/containers/image/v5 v5.6.0 => github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0 github.com/containers/image/v5/copy github.com/containers/image/v5/directory github.com/containers/image/v5/directory/explicitfilepath |