diff options
Diffstat (limited to 'pkg')
77 files changed, 3620 insertions, 404 deletions
diff --git a/pkg/api/handlers/compat/containers_prune.go b/pkg/api/handlers/compat/containers_prune.go index e37929d27..61ea7a89e 100644 --- a/pkg/api/handlers/compat/containers_prune.go +++ b/pkg/api/handlers/compat/containers_prune.go @@ -23,7 +23,7 @@ func PruneContainers(w http.ResponseWriter, r *http.Request) { filterFuncs := make([]libpod.ContainerFilter, 0, len(*filtersMap)) for k, v := range *filtersMap { - generatedFunc, err := filters.GenerateContainerFilterFuncs(k, v, runtime) + generatedFunc, err := filters.GeneratePruneContainerFilterFuncs(k, v, runtime) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/compat/containers_top.go b/pkg/api/handlers/compat/containers_top.go index ab9f613af..cae044a89 100644 --- a/pkg/api/handlers/compat/containers_top.go +++ b/pkg/api/handlers/compat/containers_top.go @@ -46,8 +46,16 @@ func TopContainer(w http.ResponseWriter, r *http.Request) { var body = handlers.ContainerTopOKBody{} if len(output) > 0 { body.Titles = strings.Split(output[0], "\t") + for i := range body.Titles { + body.Titles[i] = strings.TrimSpace(body.Titles[i]) + } + for _, line := range output[1:] { - body.Processes = append(body.Processes, strings.Split(line, "\t")) + process := strings.Split(line, "\t") + for i := range process { + process[i] = strings.TrimSpace(process[i]) + } + body.Processes = append(body.Processes, process) } } utils.WriteJSON(w, http.StatusOK, body) diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 7751b91a7..6dac789d6 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -13,7 +13,7 @@ import ( "time" "github.com/containers/buildah" - "github.com/containers/buildah/imagebuildah" + buildahDefine "github.com/containers/buildah/define" "github.com/containers/buildah/util" "github.com/containers/image/v5/types" "github.com/containers/podman/v3/libpod" @@ -69,7 +69,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { BuildArgs string `schema:"buildargs"` CacheFrom string `schema:"cachefrom"` Compression uint64 `schema:"compression"` - ConfigureNetwork int64 `schema:"networkmode"` + ConfigureNetwork string `schema:"networkmode"` CpuPeriod uint64 `schema:"cpuperiod"` // nolint CpuQuota int64 `schema:"cpuquota"` // nolint CpuSetCpus string `schema:"cpusetcpus"` // nolint @@ -84,7 +84,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { ForceRm bool `schema:"forcerm"` From string `schema:"from"` HTTPProxy bool `schema:"httpproxy"` - Isolation int64 `schema:"isolation"` + Isolation string `schema:"isolation"` Ignore bool `schema:"ignore"` Jobs int `schema:"jobs"` // nolint Labels string `schema:"labels"` @@ -98,6 +98,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { OutputFormat string `schema:"outputformat"` Platform string `schema:"platform"` Pull bool `schema:"pull"` + PullPolicy string `schema:"pullpolicy"` Quiet bool `schema:"q"` Registry string `schema:"registry"` Rm bool `schema:"rm"` @@ -199,15 +200,17 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } format := buildah.Dockerv2ImageManifest registry := query.Registry - isolation := buildah.IsolationChroot - /* - // FIXME, This is very broken. Buildah will only work with chroot - isolation := buildah.IsolationDefault - */ + isolation := buildah.IsolationDefault if utils.IsLibpodRequest(r) { - // isolation = buildah.Isolation(query.Isolation) + isolation = parseLibPodIsolation(query.Isolation) registry = "" format = query.OutputFormat + } else { + if _, found := r.URL.Query()["isolation"]; found { + if query.Isolation != "" && query.Isolation != "default" { + logrus.Debugf("invalid `isolation` parameter: %q", query.Isolation) + } + } } var additionalTags []string if len(query.Tag) > 1 { @@ -273,10 +276,14 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { jobs = query.Jobs } - pullPolicy := buildah.PullIfMissing - if _, found := r.URL.Query()["pull"]; found { - if query.Pull { - pullPolicy = buildah.PullAlways + pullPolicy := buildahDefine.PullIfMissing + if utils.IsLibpodRequest(r) { + pullPolicy = buildahDefine.PolicyMap[query.PullPolicy] + } else { + if _, found := r.URL.Query()["pull"]; found { + if query.Pull { + pullPolicy = buildahDefine.PullAlways + } } } @@ -307,7 +314,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "Decode()")) return } - buildOptions := imagebuildah.BuildOptions{ + buildOptions := buildahDefine.BuildOptions{ AddCapabilities: addCaps, AdditionalTags: additionalTags, Annotations: annotations, @@ -329,7 +336,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { CNIConfigDir: rtc.Network.CNIPluginDirs[0], CNIPluginPath: util.DefaultCNIPluginPath, Compression: compression, - ConfigureNetwork: buildah.NetworkConfigurationPolicy(query.ConfigureNetwork), + ConfigureNetwork: parseNetworkConfigurationPolicy(query.ConfigureNetwork), ContextDirectory: contextDirectory, Devices: devices, DropCapabilities: dropCaps, @@ -342,6 +349,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { Jobs: &jobs, Labels: labels, Layers: query.Layers, + LogRusage: query.LogRusage, Manifest: query.Manifest, MaxPullPushRetries: 3, NamespaceOptions: nsoptions, @@ -459,6 +467,40 @@ loop: } } +func parseNetworkConfigurationPolicy(network string) buildah.NetworkConfigurationPolicy { + if val, err := strconv.Atoi(network); err == nil { + return buildah.NetworkConfigurationPolicy(val) + } + switch network { + case "NetworkDefault": + return buildah.NetworkDefault + case "NetworkDisabled": + return buildah.NetworkDisabled + case "NetworkEnabled": + return buildah.NetworkEnabled + default: + return buildah.NetworkDefault + } +} + +func parseLibPodIsolation(isolation string) buildah.Isolation { // nolint + if val, err := strconv.Atoi(isolation); err == nil { + return buildah.Isolation(val) + } + switch isolation { + case "IsolationDefault", "default": + return buildah.IsolationDefault + case "IsolationOCI": + return buildah.IsolationOCI + case "IsolationChroot": + return buildah.IsolationChroot + case "IsolationOCIRootless": + return buildah.IsolationOCIRootless + default: + return buildah.IsolationDefault + } +} + func extractTarFile(r *http.Request) (string, error) { // build a home for the request body anchorDir, err := ioutil.TempDir("", "libpod_builder") diff --git a/pkg/api/handlers/compat/images_prune.go b/pkg/api/handlers/compat/images_prune.go index 63daaa780..ddf559ec6 100644 --- a/pkg/api/handlers/compat/images_prune.go +++ b/pkg/api/handlers/compat/images_prune.go @@ -8,8 +8,8 @@ import ( "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/pkg/api/handlers" "github.com/containers/podman/v3/pkg/api/handlers/utils" + "github.com/containers/podman/v3/pkg/util" "github.com/docker/docker/api/types" - "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -17,27 +17,20 @@ func PruneImages(w http.ResponseWriter, r *http.Request) { var ( filters []string ) - decoder := r.Context().Value("decoder").(*schema.Decoder) runtime := r.Context().Value("runtime").(*libpod.Runtime) - query := struct { - All bool - Filters map[string][]string `schema:"filters"` - }{ - // This is where you can override the golang default value for one of fields - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - for k, v := range query.Filters { + for k, v := range *filterMap { for _, val := range v { filters = append(filters, fmt.Sprintf("%s=%s", k, val)) } } - imagePruneReports, err := runtime.ImageRuntime().PruneImages(r.Context(), query.All, filters) + imagePruneReports, err := runtime.ImageRuntime().PruneImages(r.Context(), false, filters) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/compat/images_remove.go b/pkg/api/handlers/compat/images_remove.go index 874c57f16..e89558a86 100644 --- a/pkg/api/handlers/compat/images_remove.go +++ b/pkg/api/handlers/compat/images_remove.go @@ -4,7 +4,10 @@ import ( "net/http" "github.com/containers/podman/v3/libpod" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/api/handlers/utils" + "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -30,28 +33,32 @@ func RemoveImage(w http.ResponseWriter, r *http.Request) { } } name := utils.GetName(r) - newImage, err := runtime.ImageRuntime().NewFromLocal(name) - if err != nil { - utils.ImageNotFound(w, name, errors.Wrapf(err, "failed to find image %s", name)) - return + imageEngine := abi.ImageEngine{Libpod: runtime} + + options := entities.ImageRemoveOptions{ + Force: query.Force, } + report, rmerrors := imageEngine.Remove(r.Context(), []string{name}, options) + if len(rmerrors) > 0 && rmerrors[0] != nil { + err := rmerrors[0] + if errors.Cause(err) == define.ErrNoSuchImage { + utils.ImageNotFound(w, name, errors.Wrapf(err, "failed to find image %s", name)) + return + } - results, err := runtime.RemoveImage(r.Context(), newImage, query.Force) - if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) return } - - response := make([]map[string]string, 0, len(results.Untagged)+1) - deleted := make(map[string]string, 1) - deleted["Deleted"] = results.Deleted - response = append(response, deleted) - - for _, u := range results.Untagged { + response := make([]map[string]string, 0, len(report.Untagged)+1) + for _, d := range report.Deleted { + deleted := make(map[string]string, 1) + deleted["Deleted"] = d + response = append(response, deleted) + } + for _, u := range report.Untagged { untagged := make(map[string]string, 1) untagged["Untagged"] = u response = append(response, untagged) } - utils.WriteResponse(w, http.StatusOK, response) } diff --git a/pkg/api/handlers/compat/resize.go b/pkg/api/handlers/compat/resize.go index 1bf7ad460..23ed33a22 100644 --- a/pkg/api/handlers/compat/resize.go +++ b/pkg/api/handlers/compat/resize.go @@ -19,8 +19,9 @@ func ResizeTTY(w http.ResponseWriter, r *http.Request) { // /containers/{id}/resize query := struct { - Height uint16 `schema:"h"` - Width uint16 `schema:"w"` + Height uint16 `schema:"h"` + Width uint16 `schema:"w"` + IgnoreNotRunning bool `schema:"running"` }{ // override any golang type defaults } @@ -48,14 +49,17 @@ func ResizeTTY(w http.ResponseWriter, r *http.Request) { if state, err := ctnr.State(); err != nil { utils.InternalServerError(w, errors.Wrapf(err, "cannot obtain container state")) return - } else if state != define.ContainerStateRunning { + } else if state != define.ContainerStateRunning && !query.IgnoreNotRunning { utils.Error(w, "Container not running", http.StatusConflict, fmt.Errorf("container %q in wrong state %q", name, state.String())) return } + // If container is not running, ignore since this can be a race condition, and is expected if err := ctnr.AttachResize(sz); err != nil { - utils.InternalServerError(w, errors.Wrapf(err, "cannot resize container")) - return + if errors.Cause(err) != define.ErrCtrStateInvalid || !query.IgnoreNotRunning { + utils.InternalServerError(w, errors.Wrapf(err, "cannot resize container")) + return + } } // This is not a 204, even though we write nothing, for compatibility // reasons. @@ -70,14 +74,16 @@ func ResizeTTY(w http.ResponseWriter, r *http.Request) { if state, err := ctnr.State(); err != nil { utils.InternalServerError(w, errors.Wrapf(err, "cannot obtain session container state")) return - } else if state != define.ContainerStateRunning { + } else if state != define.ContainerStateRunning && !query.IgnoreNotRunning { utils.Error(w, "Container not running", http.StatusConflict, fmt.Errorf("container %q in wrong state %q", name, state.String())) return } if err := ctnr.ExecResize(name, sz); err != nil { - utils.InternalServerError(w, errors.Wrapf(err, "cannot resize session")) - return + if errors.Cause(err) != define.ErrCtrStateInvalid || !query.IgnoreNotRunning { + utils.InternalServerError(w, errors.Wrapf(err, "cannot resize session")) + return + } } // This is not a 204, even though we write nothing, for compatibility // reasons. diff --git a/pkg/api/handlers/compat/version.go b/pkg/api/handlers/compat/version.go index fae147440..f1cd77a9a 100644 --- a/pkg/api/handlers/compat/version.go +++ b/pkg/api/handlers/compat/version.go @@ -10,8 +10,8 @@ import ( "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/pkg/domain/entities/types" "github.com/containers/podman/v3/version" - docker "github.com/docker/docker/api/types" "github.com/pkg/errors" ) @@ -32,7 +32,7 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { return } - components := []docker.ComponentVersion{{ + components := []types.ComponentVersion{{ Name: "Podman Engine", Version: versionInfo.Version, Details: map[string]string{ @@ -52,7 +52,7 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { minVersion := version.APIVersion[version.Compat][version.MinimalAPI] utils.WriteResponse(w, http.StatusOK, entities.ComponentVersion{ - Version: docker.Version{ + Version: types.Version{ Platform: struct { Name string }{ diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index 1f306a533..158babcdc 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -22,6 +22,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/containers/podman/v3/pkg/errorhandling" + "github.com/containers/podman/v3/pkg/util" utils2 "github.com/containers/podman/v3/utils" "github.com/gorilla/schema" "github.com/pkg/errors" @@ -125,31 +126,32 @@ func PruneImages(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) query := struct { - All bool `schema:"all"` - Filters map[string][]string `schema:"filters"` + All bool `schema:"all"` }{ // override any golang type defaults } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + filterMap, err := util.PrepareFilters(r) + + if dErr := decoder.Decode(&query, r.URL.Query()); dErr != nil || err != nil { + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } var libpodFilters = []string{} if _, found := r.URL.Query()["filters"]; found { - dangling := query.Filters["all"] + dangling := (*filterMap)["all"] if len(dangling) > 0 { - query.All, err = strconv.ParseBool(query.Filters["all"][0]) + query.All, err = strconv.ParseBool((*filterMap)["all"][0]) if err != nil { utils.InternalServerError(w, err) return } } // dangling is special and not implemented in the libpod side of things - delete(query.Filters, "dangling") - for k, v := range query.Filters { + delete(*filterMap, "dangling") + for k, v := range *filterMap { libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", k, v[0])) } } diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 89a4a451f..4dc8740e2 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -44,13 +44,9 @@ func PodCreate(w http.ResponseWriter, r *http.Request) { func Pods(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - decoder := r.Context().Value("decoder").(*schema.Decoder) - query := struct { - Filters map[string][]string `schema:"filters"` - }{ - // override any golang type defaults - } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { + + filterMap, err := util.PrepareFilters(r) + if err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return @@ -58,7 +54,7 @@ func Pods(w http.ResponseWriter, r *http.Request) { containerEngine := abi.ContainerEngine{Libpod: runtime} podPSOptions := entities.PodPSOptions{ - Filters: query.Filters, + Filters: *filterMap, } pods, err := containerEngine.PodPs(r.Context(), podPSOptions) if err != nil { diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index f5eaf6f6d..736203171 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -28,10 +28,6 @@ type ContainerConfig struct { dockerContainer.Config } -type LibpodImagesLoadReport struct { - ID string `json:"id"` -} - type LibpodImagesPullReport struct { entities.ImagePullReport } diff --git a/pkg/api/handlers/types/types.go b/pkg/api/handlers/types/types.go new file mode 100644 index 000000000..71165364f --- /dev/null +++ b/pkg/api/handlers/types/types.go @@ -0,0 +1,23 @@ +package types + +import ( + "github.com/containers/podman/v3/pkg/domain/entities" +) + +// LibpodImagesRemoveReport is the return type for image removal via the rest +// api. +type LibpodImagesRemoveReport struct { + entities.ImageRemoveReport + // Image removal requires is to return data and an error. + Errors []string +} + +// HistoryResponse provides details on image layers +type HistoryResponse struct { + ID string `json:"Id"` + Created int64 + CreatedBy string + Tags []string + Size int64 + Comment string +} diff --git a/pkg/api/handlers/utils/images.go b/pkg/api/handlers/utils/images.go index 743629db8..da3c9e985 100644 --- a/pkg/api/handlers/utils/images.go +++ b/pkg/api/handlers/utils/images.go @@ -11,6 +11,7 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/image" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -58,13 +59,17 @@ func GetImages(w http.ResponseWriter, r *http.Request) ([]*image.Image, error) { runtime := r.Context().Value("runtime").(*libpod.Runtime) query := struct { All bool - Filters map[string][]string `schema:"filters"` Digests bool Filter string // Docker 1.24 compatibility }{ // This is where you can override the golang default value for one of fields } + filterMap, err := util.PrepareFilters(r) + if err != nil { + return nil, err + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { return nil, err } @@ -72,12 +77,9 @@ func GetImages(w http.ResponseWriter, r *http.Request) ([]*image.Image, error) { if _, found := r.URL.Query()["digests"]; found && query.Digests { UnSupportedParameter("digests") } - var ( - images []*image.Image - err error - ) + var images []*image.Image - queryFilters := query.Filters + queryFilters := *filterMap if !IsLibpodRequest(r) && len(query.Filter) > 0 { // Docker 1.24 compatibility if queryFilters == nil { queryFilters = make(map[string][]string) diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go index 31196aa9e..b379d52ce 100644 --- a/pkg/api/server/register_containers.go +++ b/pkg/api/server/register_containers.go @@ -587,6 +587,11 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // type: integer // required: false // description: Width to set for the terminal, in characters + // - in: query + // name: running + // type: boolean + // required: false + // description: Ignore containers not running errors // produces: // - application/json // responses: diff --git a/pkg/api/server/register_exec.go b/pkg/api/server/register_exec.go index 0f8c827c8..de437ab1a 100644 --- a/pkg/api/server/register_exec.go +++ b/pkg/api/server/register_exec.go @@ -136,6 +136,11 @@ func (s *APIServer) registerExecHandlers(r *mux.Router) error { // name: w // type: integer // description: Width of the TTY session in characters + // - in: query + // name: running + // type: boolean + // required: false + // description: Ignore containers not running errors // produces: // - application/json // responses: diff --git a/pkg/api/server/swagger.go b/pkg/api/server/swagger.go index 12fd083bb..d282edf23 100644 --- a/pkg/api/server/swagger.go +++ b/pkg/api/server/swagger.go @@ -205,7 +205,7 @@ type swagHealthCheckRunResponse struct { type swagVersion struct { // in:body Body struct { - entities.SystemVersionReport + entities.ComponentVersion } } diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index 21a8e7a8b..fd93c5ac7 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -22,14 +22,6 @@ import ( "golang.org/x/crypto/ssh/agent" ) -var ( - BasePath = &url.URL{ - Scheme: "http", - Host: "d", - Path: "/v" + version.APIVersion[version.Libpod][version.CurrentAPI].String() + "/libpod", - } -) - type APIResponse struct { *http.Response Request *http.Request @@ -318,16 +310,24 @@ func (c *Connection) DoRequest(httpBody io.Reader, httpMethod, endpoint string, err error response *http.Response ) - safePathValues := make([]interface{}, len(pathValues)) - // Make sure path values are http url safe + + params := make([]interface{}, len(pathValues)+3) + + // Including the semver suffices breaks older services... so do not include them + v := version.APIVersion[version.Libpod][version.CurrentAPI] + params[0] = v.Major + params[1] = v.Minor + params[2] = v.Patch for i, pv := range pathValues { - safePathValues[i] = url.PathEscape(pv) + // url.URL lacks the semantics for escaping embedded path parameters... so we manually + // escape each one and assume the caller included the correct formatting in "endpoint" + params[i+3] = url.PathEscape(pv) } - // Lets eventually use URL for this which might lead to safer - // usage - safeEndpoint := fmt.Sprintf(endpoint, safePathValues...) - e := BasePath.String() + safeEndpoint - req, err := http.NewRequest(httpMethod, e, httpBody) + + uri := fmt.Sprintf("http://d/v%d.%d.%d/libpod"+endpoint, params...) + logrus.Debugf("DoRequest Method: %s URI: %v", httpMethod, uri) + + req, err := http.NewRequest(httpMethod, uri, httpBody) if err != nil { return nil, err } diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go index f48b99a95..fd8a7011d 100644 --- a/pkg/bindings/containers/attach.go +++ b/pkg/bindings/containers/attach.go @@ -307,6 +307,7 @@ func resizeTTY(ctx context.Context, endpoint string, height *int, width *int) er if width != nil { params.Set("w", strconv.Itoa(*width)) } + params.Set("running", "true") rsp, err := conn.DoRequest(nil, http.MethodPost, endpoint, params, nil) if err != nil { return err @@ -336,7 +337,7 @@ func attachHandleResize(ctx, winCtx context.Context, winChange chan os.Signal, i case <-winCtx.Done(): return case <-winChange: - h, w, err := terminal.GetSize(int(file.Fd())) + w, h, err := terminal.GetSize(int(file.Fd())) if err != nil { logrus.Warnf("failed to obtain TTY size: %v", err) } diff --git a/pkg/bindings/containers/types.go b/pkg/bindings/containers/types.go index 2d0e65bb4..f63e35bf1 100644 --- a/pkg/bindings/containers/types.go +++ b/pkg/bindings/containers/types.go @@ -210,8 +210,9 @@ type RenameOptions struct { // ResizeTTYOptions are optional options for resizing // container TTYs type ResizeTTYOptions struct { - Height *int - Width *int + Height *int + Width *int + Running *bool } //go:generate go run ../generator/generator.go ResizeExecTTYOptions diff --git a/pkg/bindings/containers/types_resizetty_options.go b/pkg/bindings/containers/types_resizetty_options.go index 68527b330..94946692f 100644 --- a/pkg/bindings/containers/types_resizetty_options.go +++ b/pkg/bindings/containers/types_resizetty_options.go @@ -51,3 +51,19 @@ func (o *ResizeTTYOptions) GetWidth() int { } return *o.Width } + +// WithRunning +func (o *ResizeTTYOptions) WithRunning(value bool) *ResizeTTYOptions { + v := &value + o.Running = v + return o +} + +// GetRunning +func (o *ResizeTTYOptions) GetRunning() bool { + var running bool + if o.Running == nil { + return running + } + return *o.Running +} diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 9d77883f9..c79d79136 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -15,7 +15,6 @@ import ( "strconv" "strings" - "github.com/containers/buildah" "github.com/containers/podman/v3/pkg/auth" "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/domain/entities" @@ -175,9 +174,8 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO if len(platform) > 0 { params.Set("platform", platform) } - if options.PullPolicy == buildah.PullAlways { - params.Set("pull", "1") - } + params.Set("pullpolicy", options.PullPolicy.String()) + if options.Quiet { params.Set("q", "1") } diff --git a/pkg/bindings/images/images.go b/pkg/bindings/images/images.go index 1be2bdfdd..8680d6baa 100644 --- a/pkg/bindings/images/images.go +++ b/pkg/bindings/images/images.go @@ -8,7 +8,7 @@ import ( "net/url" "strconv" - "github.com/containers/podman/v3/pkg/api/handlers" + "github.com/containers/podman/v3/pkg/api/handlers/types" "github.com/containers/podman/v3/pkg/auth" "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/domain/entities" @@ -96,12 +96,12 @@ func Tree(ctx context.Context, nameOrID string, options *TreeOptions) (*entities } // History returns the parent layers of an image. -func History(ctx context.Context, nameOrID string, options *HistoryOptions) ([]*handlers.HistoryResponse, error) { +func History(ctx context.Context, nameOrID string, options *HistoryOptions) ([]*types.HistoryResponse, error) { if options == nil { options = new(HistoryOptions) } _ = options - var history []*handlers.HistoryResponse + var history []*types.HistoryResponse conn, err := bindings.GetClient(ctx) if err != nil { return nil, err diff --git a/pkg/bindings/images/rm.go b/pkg/bindings/images/rm.go index beecce7bf..e45e583f4 100644 --- a/pkg/bindings/images/rm.go +++ b/pkg/bindings/images/rm.go @@ -4,7 +4,7 @@ import ( "context" "net/http" - "github.com/containers/podman/v3/pkg/api/handlers" + "github.com/containers/podman/v3/pkg/api/handlers/types" "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/errorhandling" @@ -19,7 +19,7 @@ func Remove(ctx context.Context, images []string, options *RemoveOptions) (*enti // FIXME - bindings tests are missing for this endpoint. Once the CI is // re-enabled for bindings, we need to add them. At the time of writing, // the tests don't compile. - var report handlers.LibpodImagesRemoveReport + var report types.LibpodImagesRemoveReport conn, err := bindings.GetClient(ctx) if err != nil { return nil, []error{err} diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 7bf70c82b..1f3e46729 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -1,7 +1,7 @@ package images import ( - "github.com/containers/buildah/imagebuildah" + buildahDefine "github.com/containers/buildah/define" ) //go:generate go run ../generator/generator.go RemoveOptions @@ -162,7 +162,7 @@ type PullOptions struct { //BuildOptions are optional options for building images type BuildOptions struct { - imagebuildah.BuildOptions + buildahDefine.BuildOptions } //go:generate go run ../generator/generator.go ExistsOptions diff --git a/pkg/bindings/test/containers_test.go b/pkg/bindings/test/containers_test.go index b0ddc7862..4f049d18b 100644 --- a/pkg/bindings/test/containers_test.go +++ b/pkg/bindings/test/containers_test.go @@ -550,26 +550,24 @@ var _ = Describe("Podman containers ", func() { filtersIncorrect := map[string][]string{ "status": {"dummy"}, } - pruneResponse, err := containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filtersIncorrect)) + _, err = containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filtersIncorrect)) Expect(err).ToNot(BeNil()) - // Mismatched filter params no container should be pruned. + // List filter params should not work with prune. filtersIncorrect = map[string][]string{ - "name": {"r"}, + "name": {"top"}, } - pruneResponse, err = containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filtersIncorrect)) - Expect(err).To(BeNil()) - Expect(len(reports.PruneReportsIds(pruneResponse))).To(Equal(0)) - Expect(len(reports.PruneReportsErrs(pruneResponse))).To(Equal(0)) + _, err = containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filtersIncorrect)) + Expect(err).ToNot(BeNil()) - // Valid filter params container should be pruned now. - filters := map[string][]string{ - "name": {"top"}, + // Mismatched filter params no container should be pruned. + filtersIncorrect = map[string][]string{ + "label": {"xyz"}, } - pruneResponse, err = containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filters)) + pruneResponse, err := containers.Prune(bt.conn, new(containers.PruneOptions).WithFilters(filtersIncorrect)) Expect(err).To(BeNil()) + Expect(len(reports.PruneReportsIds(pruneResponse))).To(Equal(0)) Expect(len(reports.PruneReportsErrs(pruneResponse))).To(Equal(0)) - Expect(len(reports.PruneReportsIds(pruneResponse))).To(Equal(1)) }) It("podman prune running containers", func() { diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 7999d8209..3cc46ed0a 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -8,33 +8,32 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/podman/v3/pkg/inspect" "github.com/containers/podman/v3/pkg/trust" - docker "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/opencontainers/go-digest" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) type Image struct { - ID string `json:"Id"` - RepoTags []string `json:",omitempty"` - RepoDigests []string `json:",omitempty"` - Parent string `json:",omitempty"` - Comment string `json:",omitempty"` - Created string `json:",omitempty"` - Container string `json:",omitempty"` - ContainerConfig *container.Config `json:",omitempty"` - DockerVersion string `json:",omitempty"` - Author string `json:",omitempty"` - Config *container.Config `json:",omitempty"` - Architecture string `json:",omitempty"` - Variant string `json:",omitempty"` - Os string `json:",omitempty"` - OsVersion string `json:",omitempty"` - Size int64 `json:",omitempty"` - VirtualSize int64 `json:",omitempty"` - GraphDriver docker.GraphDriverData `json:",omitempty"` - RootFS docker.RootFS `json:",omitempty"` - Metadata docker.ImageMetadata `json:",omitempty"` + ID string `json:"Id"` + RepoTags []string `json:",omitempty"` + RepoDigests []string `json:",omitempty"` + Parent string `json:",omitempty"` + Comment string `json:",omitempty"` + Created string `json:",omitempty"` + Container string `json:",omitempty"` + ContainerConfig *container.Config `json:",omitempty"` + DockerVersion string `json:",omitempty"` + Author string `json:",omitempty"` + Config *container.Config `json:",omitempty"` + Architecture string `json:",omitempty"` + Variant string `json:",omitempty"` + Os string `json:",omitempty"` + OsVersion string `json:",omitempty"` + Size int64 `json:",omitempty"` + VirtualSize int64 `json:",omitempty"` + GraphDriver string `json:",omitempty"` + RootFS string `json:",omitempty"` + Metadata string `json:",omitempty"` // Podman extensions Digest digest.Digest `json:",omitempty"` diff --git a/pkg/domain/entities/system.go b/pkg/domain/entities/system.go index 4b8383613..1a671d59e 100644 --- a/pkg/domain/entities/system.go +++ b/pkg/domain/entities/system.go @@ -5,7 +5,7 @@ import ( "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities/reports" - "github.com/docker/docker/api/types" + "github.com/containers/podman/v3/pkg/domain/entities/types" "github.com/spf13/cobra" ) diff --git a/pkg/domain/entities/types.go b/pkg/domain/entities/types.go index 9efc64c93..02e374111 100644 --- a/pkg/domain/entities/types.go +++ b/pkg/domain/entities/types.go @@ -3,7 +3,7 @@ package entities import ( "net" - "github.com/containers/buildah/imagebuildah" + buildahDefine "github.com/containers/buildah/define" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/storage/pkg/archive" @@ -91,7 +91,7 @@ type ContainerCreateResponse struct { // BuildOptions describe the options for building container images. type BuildOptions struct { - imagebuildah.BuildOptions + buildahDefine.BuildOptions } // BuildReport is the image-build report. diff --git a/pkg/domain/entities/types/auth.go b/pkg/domain/entities/types/auth.go new file mode 100644 index 000000000..ddf15bb18 --- /dev/null +++ b/pkg/domain/entities/types/auth.go @@ -0,0 +1,22 @@ +package types // import "github.com/docker/docker/api/types" + +// AuthConfig contains authorization information for connecting to a Registry +type AuthConfig struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth,omitempty"` + + // Email is an optional value associated with the username. + // This field is deprecated and will be removed in a later + // version of docker. + Email string `json:"email,omitempty"` + + ServerAddress string `json:"serveraddress,omitempty"` + + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `json:"identitytoken,omitempty"` + + // RegistryToken is a bearer token to be sent to a registry + RegistryToken string `json:"registrytoken,omitempty"` +} diff --git a/pkg/domain/entities/types/types.go b/pkg/domain/entities/types/types.go new file mode 100644 index 000000000..77834c0cb --- /dev/null +++ b/pkg/domain/entities/types/types.go @@ -0,0 +1,28 @@ +package types // import "github.com/docker/docker/api/types" + +// ComponentVersion describes the version information for a specific component. +type ComponentVersion struct { + Name string + Version string + Details map[string]string `json:",omitempty"` +} + +// Version contains response of Engine API: +// GET "/version" +type Version struct { + Platform struct{ Name string } `json:",omitempty"` + Components []ComponentVersion `json:",omitempty"` + + // The following fields are deprecated, they relate to the Engine component and are kept for backwards compatibility + + Version string + APIVersion string `json:"ApiVersion"` + MinAPIVersion string `json:"MinAPIVersion,omitempty"` + GitCommit string + GoVersion string + Os string + Arch string + KernelVersion string `json:",omitempty"` + Experimental bool `json:",omitempty"` + BuildTime string `json:",omitempty"` +} diff --git a/pkg/domain/entities/volumes.go b/pkg/domain/entities/volumes.go index beb2a75ac..55a6a1b14 100644 --- a/pkg/domain/entities/volumes.go +++ b/pkg/domain/entities/volumes.go @@ -4,10 +4,72 @@ import ( "net/url" "github.com/containers/podman/v3/libpod/define" - docker_api_types "github.com/docker/docker/api/types" - docker_api_types_volume "github.com/docker/docker/api/types/volume" ) +// Volume volume +// swagger:model Volume +type volume struct { + + // Date/Time the volume was created. + CreatedAt string `json:"CreatedAt,omitempty"` + + // Name of the volume driver used by the volume. + // Required: true + Driver string `json:"Driver"` + + // User-defined key/value metadata. + // Required: true + Labels map[string]string `json:"Labels"` + + // Mount path of the volume on the host. + // Required: true + Mountpoint string `json:"Mountpoint"` + + // Name of the volume. + // Required: true + Name string `json:"Name"` + + // The driver specific options used when creating the volume. + // + // Required: true + Options map[string]string `json:"Options"` + + // The level at which the volume exists. Either `global` for cluster-wide, + // or `local` for machine level. + // + // Required: true + Scope string `json:"Scope"` + + // Low-level details about the volume, provided by the volume driver. + // Details are returned as a map with key/value pairs: + // `{"key":"value","key2":"value2"}`. + // + // The `Status` field is optional, and is omitted if the volume driver + // does not support this feature. + // + Status map[string]interface{} `json:"Status,omitempty"` + + // usage data + UsageData *VolumeUsageData `json:"UsageData,omitempty"` +} + +type VolumeUsageData struct { + + // The number of containers referencing this volume. This field + // is set to `-1` if the reference-count is not available. + // + // Required: true + RefCount int64 `json:"RefCount"` + + // Amount of disk space used by the volume (in bytes). This information + // is only available for volumes created with the `"local"` volume + // driver. For volumes created with other volume drivers, this field + // is set to `-1` ("not available") + // + // Required: true + Size int64 `json:"Size"` +} + // swagger:model VolumeCreate type VolumeCreateOptions struct { // New volume's name. Can be left blank @@ -113,14 +175,14 @@ type SwagVolumeListResponse struct { */ // swagger:model DockerVolumeCreate -type DockerVolumeCreate docker_api_types_volume.VolumeCreateBody +type DockerVolumeCreate VolumeCreateBody // This response definition is used for both the create and inspect endpoints // swagger:response DockerVolumeInfoResponse type SwagDockerVolumeInfoResponse struct { // in:body Body struct { - docker_api_types.Volume + volume } } @@ -129,6 +191,30 @@ type SwagDockerVolumeInfoResponse struct { type SwagDockerVolumePruneResponse struct { // in:body Body struct { - docker_api_types.VolumesPruneReport + // docker_api_types.VolumesPruneReport } } + +// VolumeCreateBody Volume configuration +// swagger:model VolumeCreateBody +type VolumeCreateBody struct { + + // Name of the volume driver to use. + // Required: true + Driver string `json:"Driver"` + + // A mapping of driver options and values. These options are + // passed directly to the driver and are driver specific. + // + // Required: true + DriverOpts map[string]string `json:"DriverOpts"` + + // User-defined key/value metadata. + // Required: true + Labels map[string]string `json:"Labels"` + + // The new volume's name. If not specified, Docker generates a name. + // + // Required: true + Name string `json:"Name"` +} diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index 02727e841..19d704da1 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -23,27 +23,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo case "label": // we have to match that all given labels exits on that container return func(c *libpod.Container) bool { - labels := c.Labels() - for _, filterValue := range filterValues { - matched := false - filterArray := strings.SplitN(filterValue, "=", 2) - filterKey := filterArray[0] - if len(filterArray) > 1 { - filterValue = filterArray[1] - } else { - filterValue = "" - } - for labelKey, labelValue := range labels { - if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { - matched = true - break - } - } - if !matched { - return false - } - } - return true + return util.MatchLabelFilters(filterValues, c.Labels()) }, nil case "name": // we only have to match one name @@ -185,16 +165,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo return false }, nil case "until": - until, err := util.ComputeUntilTimestamp(filter, filterValues) - if err != nil { - return nil, err - } - return func(c *libpod.Container) bool { - if !until.IsZero() && c.CreatedTime().After((until)) { - return true - } - return false - }, nil + return prepareUntilFilterFunc(filterValues) case "pod": var pods []*libpod.Pod for _, podNameOrID := range filterValues { @@ -246,3 +217,29 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo } return nil, errors.Errorf("%s is an invalid filter", filter) } + +// GeneratePruneContainerFilterFuncs return ContainerFilter functions based of filter for prune operation +func GeneratePruneContainerFilterFuncs(filter string, filterValues []string, r *libpod.Runtime) (func(container *libpod.Container) bool, error) { + switch filter { + case "label": + return func(c *libpod.Container) bool { + return util.MatchLabelFilters(filterValues, c.Labels()) + }, nil + case "until": + return prepareUntilFilterFunc(filterValues) + } + return nil, errors.Errorf("%s is an invalid filter", filter) +} + +func prepareUntilFilterFunc(filterValues []string) (func(container *libpod.Container) bool, error) { + until, err := util.ComputeUntilTimestamp(filterValues) + if err != nil { + return nil, err + } + return func(c *libpod.Container) bool { + if !until.IsZero() && c.CreatedTime().After((until)) { + return true + } + return false + }, nil +} diff --git a/pkg/domain/filters/pods.go b/pkg/domain/filters/pods.go index 0490a4848..9a1c7d19d 100644 --- a/pkg/domain/filters/pods.go +++ b/pkg/domain/filters/pods.go @@ -114,26 +114,7 @@ func GeneratePodFilterFunc(filter string, filterValues []string) ( case "label": return func(p *libpod.Pod) bool { labels := p.Labels() - for _, filterValue := range filterValues { - matched := false - filterArray := strings.SplitN(filterValue, "=", 2) - filterKey := filterArray[0] - if len(filterArray) > 1 { - filterValue = filterArray[1] - } else { - filterValue = "" - } - for labelKey, labelValue := range labels { - if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { - matched = true - break - } - } - if !matched { - return false - } - } - return true + return util.MatchLabelFilters(filterValues, labels) }, nil case "network": return func(p *libpod.Pod) bool { diff --git a/pkg/domain/filters/volumes.go b/pkg/domain/filters/volumes.go index bc1756cf5..9b2fc5280 100644 --- a/pkg/domain/filters/volumes.go +++ b/pkg/domain/filters/volumes.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/containers/podman/v3/libpod" + "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" ) @@ -29,21 +30,9 @@ func GenerateVolumeFilters(filters url.Values) ([]libpod.VolumeFilter, error) { return v.Scope() == scopeVal }) case "label": - filterArray := strings.SplitN(val, "=", 2) - filterKey := filterArray[0] - var filterVal string - if len(filterArray) > 1 { - filterVal = filterArray[1] - } else { - filterVal = "" - } + filter := val vf = append(vf, func(v *libpod.Volume) bool { - for labelKey, labelValue := range v.Labels() { - if labelKey == filterKey && (filterVal == "" || labelValue == filterVal) { - return true - } - } - return false + return util.MatchLabelFilters([]string{filter}, v.Labels()) }) case "opt": filterArray := strings.SplitN(val, "=", 2) diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 637531ee9..24261e5ed 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -194,7 +194,7 @@ func (ic *ContainerEngine) ContainerStop(ctx context.Context, namesOrIds []strin func (ic *ContainerEngine) ContainerPrune(ctx context.Context, options entities.ContainerPruneOptions) ([]*reports.PruneReport, error) { filterFuncs := make([]libpod.ContainerFilter, 0, len(options.Filters)) for k, v := range options.Filters { - generatedFunc, err := dfilters.GenerateContainerFilterFuncs(k, v, ic.Libpod) + generatedFunc, err := dfilters.GeneratePruneContainerFilterFuncs(k, v, ic.Libpod) if err != nil { return nil, err } diff --git a/pkg/domain/infra/abi/generate.go b/pkg/domain/infra/abi/generate.go index 161becbfa..94f649e15 100644 --- a/pkg/domain/infra/abi/generate.go +++ b/pkg/domain/infra/abi/generate.go @@ -44,11 +44,10 @@ func (ic *ContainerEngine) GenerateSystemd(ctx context.Context, nameOrID string, func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, options entities.GenerateKubeOptions) (*entities.GenerateKubeReport, error) { var ( pods []*libpod.Pod - podYAML *k8sAPI.Pod - err error ctrs []*libpod.Container - servicePorts []k8sAPI.ServicePort - serviceYAML k8sAPI.Service + kubePods []*k8sAPI.Pod + kubeServices []k8sAPI.Service + content []byte ) for _, nameOrID := range nameOrIDs { // Get the container in question @@ -59,9 +58,6 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return nil, err } pods = append(pods, pod) - if len(pods) > 1 { - return nil, errors.New("can only generate single pod at a time") - } } else { if len(ctr.Dependencies()) > 0 { return nil, errors.Wrapf(define.ErrNotImplemented, "containers with dependencies") @@ -79,20 +75,29 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return nil, errors.New("cannot generate pods and containers at the same time") } - if len(pods) == 1 { - podYAML, servicePorts, err = pods[0].GenerateForKube() + if len(pods) >= 1 { + pos, svcs, err := getKubePods(pods, options.Service) + if err != nil { + return nil, err + } + + kubePods = append(kubePods, pos...) + if options.Service { + kubeServices = append(kubeServices, svcs...) + } } else { - podYAML, err = libpod.GenerateForKube(ctrs) - } - if err != nil { - return nil, err - } + po, err := libpod.GenerateForKube(ctrs) + if err != nil { + return nil, err + } - if options.Service { - serviceYAML = libpod.GenerateKubeServiceFromV1Pod(podYAML, servicePorts) + kubePods = append(kubePods, po) + if options.Service { + kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, []k8sAPI.ServicePort{})) + } } - content, err := generateKubeOutput(podYAML, &serviceYAML, options.Service) + content, err := generateKubeOutput(kubePods, kubeServices, options.Service) if err != nil { return nil, err } @@ -100,24 +105,56 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return &entities.GenerateKubeReport{Reader: bytes.NewReader(content)}, nil } -func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasService bool) ([]byte, error) { - var ( - output []byte - marshalledPod []byte - marshalledService []byte - err error - ) +func getKubePods(pods []*libpod.Pod, getService bool) ([]*k8sAPI.Pod, []k8sAPI.Service, error) { + kubePods := make([]*k8sAPI.Pod, 0) + kubeServices := make([]k8sAPI.Service, 0) - marshalledPod, err = yaml.Marshal(podYAML) - if err != nil { - return nil, err + for _, p := range pods { + po, svc, err := p.GenerateForKube() + if err != nil { + return nil, nil, err + } + + kubePods = append(kubePods, po) + if getService { + kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, svc)) + } } - if hasService { - marshalledService, err = yaml.Marshal(serviceYAML) + return kubePods, kubeServices, nil +} + +func generateKubeOutput(kubePods []*k8sAPI.Pod, kubeServices []k8sAPI.Service, hasService bool) ([]byte, error) { + output := make([]byte, 0) + marshalledPods := make([]byte, 0) + marshalledServices := make([]byte, 0) + + for i, p := range kubePods { + if i != 0 { + marshalledPods = append(marshalledPods, []byte("---\n")...) + } + + b, err := yaml.Marshal(p) if err != nil { return nil, err } + + marshalledPods = append(marshalledPods, b...) + } + + if hasService { + for i, s := range kubeServices { + if i != 0 { + marshalledServices = append(marshalledServices, []byte("---\n")...) + } + + b, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + + marshalledServices = append(marshalledServices, b...) + } } header := `# Generation of Kubernetes YAML is still under development! @@ -133,11 +170,12 @@ func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasSer } output = append(output, []byte(fmt.Sprintf(header, podmanVersion.Version))...) - output = append(output, marshalledPod...) + // kube generate order is based on helm install order (service, pod...) if hasService { + output = append(output, marshalledServices...) output = append(output, []byte("---\n")...) - output = append(output, marshalledService...) } + output = append(output, marshalledPods...) return output, nil } diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index efc7c86e3..3b5c141d7 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -1,6 +1,7 @@ package abi import ( + "bytes" "context" "fmt" "io" @@ -8,6 +9,7 @@ import ( "os" "strings" + "github.com/containers/common/pkg/secrets" "github.com/containers/image/v5/types" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" @@ -20,46 +22,79 @@ import ( "github.com/ghodss/yaml" "github.com/pkg/errors" "github.com/sirupsen/logrus" + yamlv3 "gopkg.in/yaml.v3" v1apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) func (ic *ContainerEngine) PlayKube(ctx context.Context, path string, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { - var ( - kubeObject v1.ObjectReference - ) + report := &entities.PlayKubeReport{} + validKinds := 0 + // read yaml document content, err := ioutil.ReadFile(path) if err != nil { return nil, err } - if err := yaml.Unmarshal(content, &kubeObject); err != nil { - return nil, errors.Wrapf(err, "unable to read %q as YAML", path) + // split yaml document + documentList, err := splitMultiDocYAML(content) + if err != nil { + return nil, err } - // NOTE: pkg/bindings/play is also parsing the file. - // A pkg/kube would be nice to refactor and abstract - // parts of the K8s-related code. - switch kubeObject.Kind { - case "Pod": - var podYAML v1.Pod - var podTemplateSpec v1.PodTemplateSpec - if err := yaml.Unmarshal(content, &podYAML); err != nil { - return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path) + // create pod on each document if it is a pod or deployment + // any other kube kind will be skipped + for _, document := range documentList { + kind, err := getKubeKind(document) + if err != nil { + return nil, errors.Wrapf(err, "unable to read %q as kube YAML", path) } - podTemplateSpec.ObjectMeta = podYAML.ObjectMeta - podTemplateSpec.Spec = podYAML.Spec - return ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options) - case "Deployment": - var deploymentYAML v1apps.Deployment - if err := yaml.Unmarshal(content, &deploymentYAML); err != nil { - return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path) + + switch kind { + case "Pod": + var podYAML v1.Pod + var podTemplateSpec v1.PodTemplateSpec + + if err := yaml.Unmarshal(document, &podYAML); err != nil { + return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path) + } + + podTemplateSpec.ObjectMeta = podYAML.ObjectMeta + podTemplateSpec.Spec = podYAML.Spec + + r, err := ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options) + if err != nil { + return nil, err + } + + report.Pods = append(report.Pods, r.Pods...) + validKinds++ + case "Deployment": + var deploymentYAML v1apps.Deployment + + if err := yaml.Unmarshal(document, &deploymentYAML); err != nil { + return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path) + } + + r, err := ic.playKubeDeployment(ctx, &deploymentYAML, options) + if err != nil { + return nil, err + } + + report.Pods = append(report.Pods, r.Pods...) + validKinds++ + default: + logrus.Infof("kube kind %s not supported", kind) + continue } - return ic.playKubeDeployment(ctx, &deploymentYAML, options) - default: - return nil, errors.Errorf("invalid YAML kind: %q. [Pod|Deployment] are the only supported Kubernetes Kinds", kubeObject.Kind) } + + if validKinds == 0 { + return nil, fmt.Errorf("YAML document does not contain any supported kube kind") + } + + return report, nil } func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAML *v1apps.Deployment, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { @@ -101,6 +136,12 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY report entities.PlayKubeReport ) + // Create the secret manager before hand + secretsManager, err := secrets.NewManager(ic.Libpod.GetSecretsStorageDir()) + if err != nil { + return nil, err + } + // check for name collision between pod and container if podName == "" { return nil, errors.Errorf("pod does not have a name") @@ -227,16 +268,17 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY } specgenOpts := kube.CtrSpecGenOptions{ - Container: container, - Image: newImage, - Volumes: volumes, - PodID: pod.ID(), - PodName: podName, - PodInfraID: podInfraID, - ConfigMaps: configMaps, - SeccompPaths: seccompPaths, - RestartPolicy: ctrRestartPolicy, - NetNSIsHost: p.NetNS.IsHost(), + Container: container, + Image: newImage, + Volumes: volumes, + PodID: pod.ID(), + PodName: podName, + PodInfraID: podInfraID, + ConfigMaps: configMaps, + SeccompPaths: seccompPaths, + RestartPolicy: ctrRestartPolicy, + NetNSIsHost: p.NetNS.IsHost(), + SecretsManager: secretsManager, } specGen, err := kube.ToSpecGen(ctx, &specgenOpts) if err != nil { @@ -290,3 +332,45 @@ func readConfigMapFromFile(r io.Reader) (v1.ConfigMap, error) { return cm, nil } + +// splitMultiDocYAML reads mutiple documents in a YAML file and +// returns them as a list. +func splitMultiDocYAML(yamlContent []byte) ([][]byte, error) { + var documentList [][]byte + + d := yamlv3.NewDecoder(bytes.NewReader(yamlContent)) + for { + var o interface{} + // read individual document + err := d.Decode(&o) + if err == io.EOF { + break + } + if err != nil { + return nil, errors.Wrapf(err, "multi doc yaml could not be split") + } + + if o != nil { + // back to bytes + document, err := yamlv3.Marshal(o) + if err != nil { + return nil, errors.Wrapf(err, "individual doc yaml could not be marshalled") + } + + documentList = append(documentList, document) + } + } + + return documentList, nil +} + +// getKubeKind unmarshals a kube YAML document and returns its kind. +func getKubeKind(obj []byte) (string, error) { + var kubeObject v1.ObjectReference + + if err := yaml.Unmarshal(obj, &kubeObject); err != nil { + return "", err + } + + return kubeObject.Kind, nil +} diff --git a/pkg/domain/infra/abi/play_test.go b/pkg/domain/infra/abi/play_test.go index 4354a3835..bbc7c3493 100644 --- a/pkg/domain/infra/abi/play_test.go +++ b/pkg/domain/infra/abi/play_test.go @@ -89,3 +89,100 @@ data: }) } } + +func TestGetKubeKind(t *testing.T) { + tests := []struct { + name string + kubeYAML string + expectError bool + expectedErrorMsg string + expected string + }{ + { + "ValidKubeYAML", + ` +apiVersion: v1 +kind: Pod +`, + false, + "", + "Pod", + }, + { + "InvalidKubeYAML", + "InvalidKubeYAML", + true, + "cannot unmarshal", + "", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + kind, err := getKubeKind([]byte(test.kubeYAML)) + if test.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.expectedErrorMsg) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, kind) + } + }) + } +} + +func TestSplitMultiDocYAML(t *testing.T) { + tests := []struct { + name string + kubeYAML string + expectError bool + expectedErrorMsg string + expected int + }{ + { + "ValidNumberOfDocs", + ` +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +`, + false, + "", + 3, + }, + { + "InvalidMultiDocYAML", + ` +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +- +`, + true, + "multi doc yaml could not be split", + 0, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + docs, err := splitMultiDocYAML([]byte(test.kubeYAML)) + if test.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.expectedErrorMsg) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, len(docs)) + } + }) + } +} diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index 9f7c8919b..a3e753384 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -67,7 +67,7 @@ func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command) if os.Geteuid() == 0 { ownsCgroup, err := cgroups.UserOwnsCurrentSystemdCgroup() if err != nil { - logrus.Warnf("Failed to detect the owner for the current cgroup: %v", err) + logrus.Infof("Failed to detect the owner for the current cgroup: %v", err) } if !ownsCgroup { conf, err := ic.Config(context.Background()) diff --git a/pkg/domain/infra/runtime_libpod.go b/pkg/domain/infra/runtime_libpod.go index 8b6581c7b..b0d9dc797 100644 --- a/pkg/domain/infra/runtime_libpod.go +++ b/pkg/domain/infra/runtime_libpod.go @@ -15,8 +15,8 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/namespaces" "github.com/containers/podman/v3/pkg/rootless" - "github.com/containers/storage" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" flag "github.com/spf13/pflag" @@ -100,7 +100,7 @@ func GetRuntimeNoStore(ctx context.Context, fs *flag.FlagSet, cfg *entities.Podm func getRuntime(ctx context.Context, fs *flag.FlagSet, opts *engineOpts) (*libpod.Runtime, error) { options := []libpod.RuntimeOption{} - storageOpts := storage.StoreOptions{} + storageOpts := types.StoreOptions{} cfg := opts.config storageSet := false @@ -237,8 +237,8 @@ func getRuntime(ctx context.Context, fs *flag.FlagSet, opts *engineOpts) (*libpo } // ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping -func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*storage.IDMappingOptions, error) { - options := storage.IDMappingOptions{ +func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*types.IDMappingOptions, error) { + options := types.IDMappingOptions{ HostUIDMapping: true, HostGIDMapping: true, } diff --git a/pkg/machine/config.go b/pkg/machine/config.go new file mode 100644 index 000000000..32b3b5c2b --- /dev/null +++ b/pkg/machine/config.go @@ -0,0 +1,132 @@ +package machine + +import ( + "net" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/containers/storage/pkg/homedir" + "github.com/pkg/errors" +) + +type InitOptions struct { + CPUS uint64 + DiskSize uint64 + IgnitionPath string + ImagePath string + IsDefault bool + Memory uint64 + Name string + URI url.URL + Username string +} + +type RemoteConnectionType string + +var ( + SSHRemoteConnection RemoteConnectionType = "ssh" + DefaultIgnitionUserName = "core" + ErrNoSuchVM = errors.New("VM does not exist") + ErrVMAlreadyExists = errors.New("VM already exists") +) + +type Download struct { + Arch string + Artifact string + CompressionType string + Format string + ImageName string `json:"image_name"` + LocalPath string + LocalUncompressedFile string + Sha256sum string + URL *url.URL + VMName string +} + +type ListOptions struct{} + +type ListResponse struct { + Name string + CreatedAt time.Time + LastUp time.Time + Running bool + VMType string +} + +type SSHOptions struct { + Args []string +} +type StartOptions struct{} + +type StopOptions struct{} + +type RemoveOptions struct { + Force bool + SaveKeys bool + SaveImage bool + SaveIgnition bool +} + +type VM interface { + Init(opts InitOptions) error + Remove(name string, opts RemoveOptions) (string, func() error, error) + SSH(name string, opts SSHOptions) error + Start(name string, opts StartOptions) error + Stop(name string, opts StopOptions) error +} + +type DistributionDownload interface { + DownloadImage() error + Get() *Download +} + +func (rc RemoteConnectionType) MakeSSHURL(host, path, port, userName string) url.URL { + userInfo := url.User(userName) + uri := url.URL{ + Scheme: "ssh", + Opaque: "", + User: userInfo, + Host: host, + Path: path, + RawPath: "", + ForceQuery: false, + RawQuery: "", + Fragment: "", + } + if len(port) > 0 { + uri.Host = net.JoinHostPort(uri.Hostname(), port) + } + return uri +} + +// GetDataDir returns the filepath where vm images should +// live for podman-machine +func GetDataDir(vmType string) (string, error) { + data, err := homedir.GetDataHome() + if err != nil { + return "", err + } + dataDir := filepath.Join(data, "containers", "podman", "machine", vmType) + if _, err := os.Stat(dataDir); !os.IsNotExist(err) { + return dataDir, nil + } + mkdirErr := os.MkdirAll(dataDir, 0755) + return dataDir, mkdirErr +} + +// GetConfigDir returns the filepath to where configuration +// files for podman-machine should live +func GetConfDir(vmType string) (string, error) { + conf, err := homedir.GetConfigHome() + if err != nil { + return "", err + } + confDir := filepath.Join(conf, "containers", "podman", "machine", vmType) + if _, err := os.Stat(confDir); !os.IsNotExist(err) { + return confDir, nil + } + mkdirErr := os.MkdirAll(confDir, 0755) + return confDir, mkdirErr +} diff --git a/pkg/machine/connection.go b/pkg/machine/connection.go new file mode 100644 index 000000000..e3985d8ac --- /dev/null +++ b/pkg/machine/connection.go @@ -0,0 +1,50 @@ +package machine + +import ( + "fmt" + + "github.com/containers/common/pkg/config" + "github.com/pkg/errors" +) + +func AddConnection(uri fmt.Stringer, name, identity string, isDefault bool) error { + if len(identity) < 1 { + return errors.New("identity must be defined") + } + cfg, err := config.ReadCustomConfig() + if err != nil { + return err + } + if _, ok := cfg.Engine.ServiceDestinations[name]; ok { + return errors.New("cannot overwrite connection") + } + if isDefault { + cfg.Engine.ActiveService = name + } + dst := config.Destination{ + URI: uri.String(), + } + dst.Identity = identity + if cfg.Engine.ServiceDestinations == nil { + cfg.Engine.ServiceDestinations = map[string]config.Destination{ + name: dst, + } + cfg.Engine.ActiveService = name + } else { + cfg.Engine.ServiceDestinations[name] = dst + } + return cfg.Write() +} + +func RemoveConnection(name string) error { + cfg, err := config.ReadCustomConfig() + if err != nil { + return err + } + if _, ok := cfg.Engine.ServiceDestinations[name]; ok { + delete(cfg.Engine.ServiceDestinations, name) + } else { + return errors.Errorf("unable to find connection named %q", name) + } + return cfg.Write() +} diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go new file mode 100644 index 000000000..32f943c87 --- /dev/null +++ b/pkg/machine/fcos.go @@ -0,0 +1,122 @@ +package machine + +import ( + "crypto/sha256" + "io/ioutil" + url2 "net/url" + "path/filepath" + "runtime" + "strings" + + digest "github.com/opencontainers/go-digest" +) + +// These should eventually be moved into machine/qemu as +// they are specific to running qemu +var ( + artifact string = "qemu" + Format string = "qcow2.xz" +) + +type FcosDownload struct { + Download +} + +func NewFcosDownloader(vmType, vmName string) (DistributionDownload, error) { + info, err := getFCOSDownload() + if err != nil { + return nil, err + } + urlSplit := strings.Split(info.Location, "/") + imageName := urlSplit[len(urlSplit)-1] + url, err := url2.Parse(info.Location) + if err != nil { + return nil, err + } + + dataDir, err := GetDataDir(vmType) + if err != nil { + return nil, err + } + + fcd := FcosDownload{ + Download: Download{ + Arch: getFcosArch(), + Artifact: artifact, + Format: Format, + ImageName: imageName, + LocalPath: filepath.Join(dataDir, imageName), + Sha256sum: info.Sha256Sum, + URL: url, + VMName: vmName, + }, + } + fcd.Download.LocalUncompressedFile = fcd.getLocalUncompressedName() + return fcd, nil +} + +func (f FcosDownload) getLocalUncompressedName() string { + uncompressedFilename := filepath.Join(filepath.Dir(f.LocalPath), f.VMName+"_"+f.ImageName) + return strings.TrimSuffix(uncompressedFilename, ".xz") +} + +func (f FcosDownload) DownloadImage() error { + // check if the latest image is already present + ok, err := UpdateAvailable(&f.Download) + if err != nil { + return err + } + if !ok { + if err := DownloadVMImage(f.URL, f.LocalPath); err != nil { + return err + } + } + return Decompress(f.LocalPath, f.getLocalUncompressedName()) +} + +func (f FcosDownload) Get() *Download { + return &f.Download +} + +type fcosDownloadInfo struct { + CompressionType string + Location string + Release string + Sha256Sum string +} + +func UpdateAvailable(d *Download) (bool, error) { + // check the sha of the local image if it exists + // get the sha of the remote image + // == dont bother to pull + files, err := ioutil.ReadDir(filepath.Dir(d.LocalPath)) + if err != nil { + return false, err + } + for _, file := range files { + if filepath.Base(d.LocalPath) == file.Name() { + b, err := ioutil.ReadFile(d.LocalPath) + if err != nil { + return false, err + } + s := sha256.Sum256(b) + sum := digest.NewDigestFromBytes(digest.SHA256, s[:]) + if sum.Encoded() == d.Sha256sum { + return true, nil + } + } + } + return false, nil +} + +func getFcosArch() string { + var arch string + // TODO fill in more architectures + switch runtime.GOARCH { + case "arm64": + arch = "aarch64" + default: + arch = "x86_64" + } + return arch +} diff --git a/pkg/machine/fcos_amd64.go b/pkg/machine/fcos_amd64.go new file mode 100644 index 000000000..36676405a --- /dev/null +++ b/pkg/machine/fcos_amd64.go @@ -0,0 +1,68 @@ +package machine + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/coreos/stream-metadata-go/fedoracoreos" + "github.com/coreos/stream-metadata-go/stream" + "github.com/sirupsen/logrus" +) + +// This should get Exported and stay put as it will apply to all fcos downloads +// getFCOS parses fedoraCoreOS's stream and returns the image download URL and the release version +func getFCOSDownload() (*fcosDownloadInfo, error) { + var ( + fcosstable stream.Stream + ) + streamurl := fedoracoreos.GetStreamURL(fedoracoreos.StreamNext) + resp, err := http.Get(streamurl.String()) + if err != nil { + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + + if err := json.Unmarshal(body, &fcosstable); err != nil { + return nil, err + } + arch, ok := fcosstable.Architectures[getFcosArch()] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no targetArch in stream") + } + artifacts := arch.Artifacts + if artifacts == nil { + return nil, fmt.Errorf("unable to pull VM image: no artifact in stream") + } + qemu, ok := artifacts[artifact] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qemu artifact in stream") + } + formats := qemu.Formats + if formats == nil { + return nil, fmt.Errorf("unable to pull VM image: no formats in stream") + } + qcow, ok := formats[Format] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qcow2.xz format in stream") + } + disk := qcow.Disk + if disk == nil { + return nil, fmt.Errorf("unable to pull VM image: no disk in stream") + } + return &fcosDownloadInfo{ + Location: disk.Location, + Release: qemu.Release, + Sha256Sum: disk.Sha256, + CompressionType: "xz", + }, nil +} diff --git a/pkg/machine/fcos_arm64.go b/pkg/machine/fcos_arm64.go new file mode 100644 index 000000000..f5cd5a505 --- /dev/null +++ b/pkg/machine/fcos_arm64.go @@ -0,0 +1,177 @@ +package machine + +import ( + "encoding/json" + "io/ioutil" + "net/http" + url2 "net/url" + + "github.com/sirupsen/logrus" +) + +const aarchBaseURL = "https://fedorapeople.org/groups/fcos-images/builds/latest/aarch64/" + +// Total hack until automation is possible. +// We need a proper json file at least to automate +func getFCOSDownload() (*fcosDownloadInfo, error) { + meta := Build{} + resp, err := http.Get(aarchBaseURL + "meta.json") + if err != nil { + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + if err := json.Unmarshal(body, &meta); err != nil { + return nil, err + } + pathURL, err := url2.Parse(meta.BuildArtifacts.Qemu.Path) + if err != nil { + return nil, err + } + + baseURL, err := url2.Parse(aarchBaseURL) + if err != nil { + return nil, err + } + pullURL := baseURL.ResolveReference(pathURL) + return &fcosDownloadInfo{ + Location: pullURL.String(), + Release: "", + Sha256Sum: meta.BuildArtifacts.Qemu.Sha256, + }, nil +} + +/* + All of this can be nuked when fcos upstream generates a proper meta data file for aarch. +*/ +type AliyunImage struct { + ImageID string `json:"id"` + Region string `json:"name"` +} + +type Amis struct { + Hvm string `json:"hvm"` + Region string `json:"name"` + Snapshot string `json:"snapshot"` +} + +type Artifact struct { + Path string `json:"path"` + Sha256 string `json:"sha256"` + SizeInBytes float64 `json:"size,omitempty"` + UncompressedSha256 string `json:"uncompressed-sha256,omitempty"` + UncompressedSize int `json:"uncompressed-size,omitempty"` +} + +type Build struct { + AlibabaAliyunUploads []AliyunImage `json:"aliyun,omitempty"` + Amis []Amis `json:"amis,omitempty"` + Architecture string `json:"coreos-assembler.basearch,omitempty"` + Azure *Cloudartifact `json:"azure,omitempty"` + BuildArtifacts *BuildArtifacts `json:"images,omitempty"` + BuildID string `json:"buildid"` + BuildRef string `json:"ref,omitempty"` + BuildSummary string `json:"summary"` + BuildTimeStamp string `json:"coreos-assembler.build-timestamp,omitempty"` + BuildURL string `json:"build-url,omitempty"` + ConfigGitRev string `json:"coreos-assembler.config-gitrev,omitempty"` + ContainerConfigGit *Git `json:"coreos-assembler.container-config-git,omitempty"` + CoreOsSource string `json:"coreos-assembler.code-source,omitempty"` + CosaContainerImageGit *Git `json:"coreos-assembler.container-image-git,omitempty"` + CosaDelayedMetaMerge bool `json:"coreos-assembler.delayed-meta-merge,omitempty"` + CosaImageChecksum string `json:"coreos-assembler.image-config-checksum,omitempty"` + CosaImageVersion int `json:"coreos-assembler.image-genver,omitempty"` + Extensions *Extensions `json:"extensions,omitempty"` + FedoraCoreOsParentCommit string `json:"fedora-coreos.parent-commit,omitempty"` + FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` + Gcp *Gcp `json:"gcp,omitempty"` + GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` + ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` + InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` + MetaStamp float64 `json:"coreos-assembler.meta-stamp,omitempty"` + Name string `json:"name"` + Oscontainer *Image `json:"oscontainer,omitempty"` + OstreeCommit string `json:"ostree-commit"` + OstreeContentBytesWritten int `json:"ostree-content-bytes-written,omitempty"` + OstreeContentChecksum string `json:"ostree-content-checksum"` + OstreeNCacheHits int `json:"ostree-n-cache-hits,omitempty"` + OstreeNContentTotal int `json:"ostree-n-content-total,omitempty"` + OstreeNContentWritten int `json:"ostree-n-content-written,omitempty"` + OstreeNMetadataTotal int `json:"ostree-n-metadata-total,omitempty"` + OstreeNMetadataWritten int `json:"ostree-n-metadata-written,omitempty"` + OstreeTimestamp string `json:"ostree-timestamp"` + OstreeVersion string `json:"ostree-version"` + OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` + PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` + PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` + ReleasePayload *Image `json:"release-payload,omitempty"` +} + +type BuildArtifacts struct { + Aliyun *Artifact `json:"aliyun,omitempty"` + Aws *Artifact `json:"aws,omitempty"` + Azure *Artifact `json:"azure,omitempty"` + AzureStack *Artifact `json:"azurestack,omitempty"` + Dasd *Artifact `json:"dasd,omitempty"` + DigitalOcean *Artifact `json:"digitalocean,omitempty"` + Exoscale *Artifact `json:"exoscale,omitempty"` + Gcp *Artifact `json:"gcp,omitempty"` + IbmCloud *Artifact `json:"ibmcloud,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Iso *Artifact `json:"iso,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` + LiveIso *Artifact `json:"live-iso,omitempty"` + LiveKernel *Artifact `json:"live-kernel,omitempty"` + LiveRootfs *Artifact `json:"live-rootfs,omitempty"` + Metal *Artifact `json:"metal,omitempty"` + Metal4KNative *Artifact `json:"metal4k,omitempty"` + OpenStack *Artifact `json:"openstack,omitempty"` + Ostree Artifact `json:"ostree"` + Qemu *Artifact `json:"qemu,omitempty"` + Vmware *Artifact `json:"vmware,omitempty"` + Vultr *Artifact `json:"vultr,omitempty"` +} + +type Cloudartifact struct { + Image string `json:"image"` + URL string `json:"url"` +} + +type Extensions struct { + Manifest map[string]interface{} `json:"manifest"` + Path string `json:"path"` + RpmOstreeState string `json:"rpm-ostree-state"` + Sha256 string `json:"sha256"` +} + +type Gcp struct { + ImageFamily string `json:"family,omitempty"` + ImageName string `json:"image"` + ImageProject string `json:"project,omitempty"` + URL string `json:"url"` +} + +type Git struct { + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Dirty string `json:"dirty,omitempty"` + Origin string `json:"origin"` +} + +type Image struct { + Comment string `json:"comment,omitempty"` + Digest string `json:"digest"` + Image string `json:"image"` +} + +type Items interface{} + +type PackageSetDifferences []Items diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go new file mode 100644 index 000000000..a68d68ac3 --- /dev/null +++ b/pkg/machine/ignition.go @@ -0,0 +1,180 @@ +package machine + +import ( + "encoding/json" + "fmt" + "io/ioutil" +) + +/* + If this file gets too nuts, we can perhaps use existing go code + to create ignition files. At this point, the file is so simple + that I chose to use structs and not import any code as I was + concerned (unsubstantiated) about too much bloat coming in. + + https://github.com/openshift/machine-config-operator/blob/master/pkg/server/server.go +*/ + +// Convenience function to convert int to ptr +func intToPtr(i int) *int { + return &i +} + +// Convenience function to convert string to ptr +func strToPtr(s string) *string { + return &s +} + +// Convenience function to convert bool to ptr +func boolToPtr(b bool) *bool { + return &b +} + +func getNodeUsr(usrName string) NodeUser { + return NodeUser{Name: &usrName} +} + +func getNodeGrp(grpName string) NodeGroup { + return NodeGroup{Name: &grpName} +} + +type DynamicIgnition struct { + Name string + Key string + VMName string + WritePath string +} + +// NewIgnitionFile +func NewIgnitionFile(ign DynamicIgnition) error { + if len(ign.Name) < 1 { + ign.Name = DefaultIgnitionUserName + } + ignVersion := Ignition{ + Version: "3.2.0", + } + + ignPassword := Passwd{ + Users: []PasswdUser{{ + Name: ign.Name, + SSHAuthorizedKeys: []SSHAuthorizedKey{SSHAuthorizedKey(ign.Key)}, + }}, + } + + ignStorage := Storage{ + Directories: getDirs(ign.Name), + Files: getFiles(ign.Name), + Links: getLinks(ign.Name), + } + + // ready is a unit file that sets up the virtual serial device + // where when the VM is done configuring, it will send an ack + // so a listening host knows it can being interacting with it + ready := `[Unit] +Requires=dev-virtio\\x2dports-%s.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo Ready >/dev/%s' +[Install] +RequiredBy=multi-user.target +` + _ = ready + ignSystemd := Systemd{ + Units: []Unit{ + { + Enabled: boolToPtr(true), + Name: "podman.socket", + }, + { + Enabled: boolToPtr(true), + Name: "ready.service", + Contents: strToPtr(fmt.Sprintf(ready, "vport1p1", "vport1p1")), + }, + }} + ignConfig := Config{ + Ignition: ignVersion, + Passwd: ignPassword, + Storage: ignStorage, + Systemd: ignSystemd, + } + b, err := json.Marshal(ignConfig) + if err != nil { + return err + } + return ioutil.WriteFile(ign.WritePath, b, 0644) +} + +func getDirs(usrName string) []Directory { + // Ignition has a bug/feature? where if you make a series of dirs + // in one swoop, then the leading dirs are creates as root. + newDirs := []string{ + "/home/" + usrName + "/.config", + "/home/" + usrName + "/.config/systemd", + "/home/" + usrName + "/.config/systemd/user", + "/home/" + usrName + "/.config/systemd/user/default.target.wants", + } + var ( + dirs = make([]Directory, len(newDirs)) + ) + for i, d := range newDirs { + newDir := Directory{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: d, + User: getNodeUsr(usrName), + }, + DirectoryEmbedded1: DirectoryEmbedded1{Mode: intToPtr(493)}, + } + dirs[i] = newDir + } + return dirs +} + +func getFiles(usrName string) []File { + var ( + files []File + ) + // Add a fake systemd service to get the user socket rolling + files = append(files, File{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/home/" + usrName + "/.config/systemd/user/linger-example.service", + User: getNodeUsr(usrName), + }, + FileEmbedded1: FileEmbedded1{ + Append: nil, + Contents: Resource{ + Source: strToPtr("data:,%5BUnit%5D%0ADescription%3DA%20systemd%20user%20unit%20demo%0AAfter%3Dnetwork-online.target%0AWants%3Dnetwork-online.target%20podman.socket%0A%5BService%5D%0AExecStart%3D%2Fusr%2Fbin%2Fsleep%20infinity%0A"), + }, + Mode: intToPtr(484), + }, + }) + + // Add a file into linger + files = append(files, File{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/var/lib/systemd/linger/core", + User: getNodeUsr(usrName), + }, + FileEmbedded1: FileEmbedded1{Mode: intToPtr(420)}, + }) + return files +} + +func getLinks(usrName string) []Link { + return []Link{{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/home/" + usrName + "/.config/systemd/user/default.target.wants/linger-example.service", + User: getNodeUsr(usrName), + }, + LinkEmbedded1: LinkEmbedded1{ + Hard: boolToPtr(false), + Target: "/home/" + usrName + "/.config/systemd/user/linger-example.service", + }, + }} +} diff --git a/pkg/machine/ignition_schema.go b/pkg/machine/ignition_schema.go new file mode 100644 index 000000000..9dbd90ba4 --- /dev/null +++ b/pkg/machine/ignition_schema.go @@ -0,0 +1,251 @@ +package machine + +/* + This file was taken from https://github.com/coreos/ignition/blob/master/config/v3_2/types/schema.go in an effort to + use more of the core-os structs but not fully commit to bringing their api in. + + // generated by "schematyper --package=types config/v3_2/schema/ignition.json -o config/v3_2/types/ignition_schema.go --root-type=Config" -- DO NOT EDIT +*/ + +type Clevis struct { + Custom *Custom `json:"custom,omitempty"` + Tang []Tang `json:"tang,omitempty"` + Threshold *int `json:"threshold,omitempty"` + Tpm2 *bool `json:"tpm2,omitempty"` +} + +type Config struct { + Ignition Ignition `json:"ignition"` + Passwd Passwd `json:"passwd,omitempty"` + Storage Storage `json:"storage,omitempty"` + Systemd Systemd `json:"systemd,omitempty"` +} + +type Custom struct { + Config string `json:"config"` + NeedsNetwork *bool `json:"needsNetwork,omitempty"` + Pin string `json:"pin"` +} + +type Device string + +type Directory struct { + Node + DirectoryEmbedded1 +} + +type DirectoryEmbedded1 struct { + Mode *int `json:"mode,omitempty"` +} + +type Disk struct { + Device string `json:"device"` + Partitions []Partition `json:"partitions,omitempty"` + WipeTable *bool `json:"wipeTable,omitempty"` +} + +type Dropin struct { + Contents *string `json:"contents,omitempty"` + Name string `json:"name"` +} + +type File struct { + Node + FileEmbedded1 +} + +type FileEmbedded1 struct { + Append []Resource `json:"append,omitempty"` + Contents Resource `json:"contents,omitempty"` + Mode *int `json:"mode,omitempty"` +} + +type Filesystem struct { + Device string `json:"device"` + Format *string `json:"format,omitempty"` + Label *string `json:"label,omitempty"` + MountOptions []MountOption `json:"mountOptions,omitempty"` + Options []FilesystemOption `json:"options,omitempty"` + Path *string `json:"path,omitempty"` + UUID *string `json:"uuid,omitempty"` + WipeFilesystem *bool `json:"wipeFilesystem,omitempty"` +} + +type FilesystemOption string + +type Group string + +type HTTPHeader struct { + Name string `json:"name"` + Value *string `json:"value,omitempty"` +} + +type HTTPHeaders []HTTPHeader + +type Ignition struct { + Config IgnitionConfig `json:"config,omitempty"` + Proxy Proxy `json:"proxy,omitempty"` + Security Security `json:"security,omitempty"` + Timeouts Timeouts `json:"timeouts,omitempty"` + Version string `json:"version,omitempty"` +} + +type IgnitionConfig struct { + Merge []Resource `json:"merge,omitempty"` + Replace Resource `json:"replace,omitempty"` +} + +type Link struct { + Node + LinkEmbedded1 +} + +type LinkEmbedded1 struct { + Hard *bool `json:"hard,omitempty"` + Target string `json:"target"` +} + +type Luks struct { + Clevis *Clevis `json:"clevis,omitempty"` + Device *string `json:"device,omitempty"` + KeyFile Resource `json:"keyFile,omitempty"` + Label *string `json:"label,omitempty"` + Name string `json:"name"` + Options []LuksOption `json:"options,omitempty"` + UUID *string `json:"uuid,omitempty"` + WipeVolume *bool `json:"wipeVolume,omitempty"` +} + +type LuksOption string + +type MountOption string + +type NoProxyItem string + +type Node struct { + Group NodeGroup `json:"group,omitempty"` + Overwrite *bool `json:"overwrite,omitempty"` + Path string `json:"path"` + User NodeUser `json:"user,omitempty"` +} + +type NodeGroup struct { + ID *int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type NodeUser struct { + ID *int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type Partition struct { + GUID *string `json:"guid,omitempty"` + Label *string `json:"label,omitempty"` + Number int `json:"number,omitempty"` + Resize *bool `json:"resize,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + SizeMiB *int `json:"sizeMiB,omitempty"` + StartMiB *int `json:"startMiB,omitempty"` + TypeGUID *string `json:"typeGuid,omitempty"` + WipePartitionEntry *bool `json:"wipePartitionEntry,omitempty"` +} + +type Passwd struct { + Groups []PasswdGroup `json:"groups,omitempty"` + Users []PasswdUser `json:"users,omitempty"` +} + +type PasswdGroup struct { + Gid *int `json:"gid,omitempty"` + Name string `json:"name"` + PasswordHash *string `json:"passwordHash,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + System *bool `json:"system,omitempty"` +} + +type PasswdUser struct { + Gecos *string `json:"gecos,omitempty"` + Groups []Group `json:"groups,omitempty"` + HomeDir *string `json:"homeDir,omitempty"` + Name string `json:"name"` + NoCreateHome *bool `json:"noCreateHome,omitempty"` + NoLogInit *bool `json:"noLogInit,omitempty"` + NoUserGroup *bool `json:"noUserGroup,omitempty"` + PasswordHash *string `json:"passwordHash,omitempty"` + PrimaryGroup *string `json:"primaryGroup,omitempty"` + SSHAuthorizedKeys []SSHAuthorizedKey `json:"sshAuthorizedKeys,omitempty"` + Shell *string `json:"shell,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + System *bool `json:"system,omitempty"` + UID *int `json:"uid,omitempty"` +} + +type Proxy struct { + HTTPProxy *string `json:"httpProxy,omitempty"` + HTTPSProxy *string `json:"httpsProxy,omitempty"` + NoProxy []NoProxyItem `json:"noProxy,omitempty"` +} + +type Raid struct { + Devices []Device `json:"devices"` + Level string `json:"level"` + Name string `json:"name"` + Options []RaidOption `json:"options,omitempty"` + Spares *int `json:"spares,omitempty"` +} + +type RaidOption string + +type Resource struct { + Compression *string `json:"compression,omitempty"` + HTTPHeaders HTTPHeaders `json:"httpHeaders,omitempty"` + Source *string `json:"source,omitempty"` + Verification Verification `json:"verification,omitempty"` +} + +type SSHAuthorizedKey string + +type Security struct { + TLS TLS `json:"tls,omitempty"` +} + +type Storage struct { + Directories []Directory `json:"directories,omitempty"` + Disks []Disk `json:"disks,omitempty"` + Files []File `json:"files,omitempty"` + Filesystems []Filesystem `json:"filesystems,omitempty"` + Links []Link `json:"links,omitempty"` + Luks []Luks `json:"luks,omitempty"` + Raid []Raid `json:"raid,omitempty"` +} + +type Systemd struct { + Units []Unit `json:"units,omitempty"` +} + +type TLS struct { + CertificateAuthorities []Resource `json:"certificateAuthorities,omitempty"` +} + +type Tang struct { + Thumbprint *string `json:"thumbprint,omitempty"` + URL string `json:"url,omitempty"` +} + +type Timeouts struct { + HTTPResponseHeaders *int `json:"httpResponseHeaders,omitempty"` + HTTPTotal *int `json:"httpTotal,omitempty"` +} + +type Unit struct { + Contents *string `json:"contents,omitempty"` + Dropins []Dropin `json:"dropins,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Mask *bool `json:"mask,omitempty"` + Name string `json:"name"` +} + +type Verification struct { + Hash *string `json:"hash,omitempty"` +} diff --git a/pkg/machine/keys.go b/pkg/machine/keys.go new file mode 100644 index 000000000..907e28f55 --- /dev/null +++ b/pkg/machine/keys.go @@ -0,0 +1,25 @@ +package machine + +import ( + "io/ioutil" + "os/exec" + "strings" +) + +// CreateSSHKeys makes a priv and pub ssh key for interacting +// the a VM. +func CreateSSHKeys(writeLocation string) (string, error) { + if err := generatekeys(writeLocation); err != nil { + return "", err + } + b, err := ioutil.ReadFile(writeLocation + ".pub") + if err != nil { + return "", err + } + return strings.TrimSuffix(string(b), "\n"), nil +} + +// generatekeys creates an ed25519 set of keys +func generatekeys(writeLocation string) error { + return exec.Command("ssh-keygen", "-N", "", "-t", "ed25519", "-f", writeLocation).Run() +} diff --git a/pkg/machine/libvirt/config.go b/pkg/machine/libvirt/config.go new file mode 100644 index 000000000..903f15fbc --- /dev/null +++ b/pkg/machine/libvirt/config.go @@ -0,0 +1,4 @@ +package libvirt + +type MachineVM struct { +} diff --git a/pkg/machine/libvirt/machine.go b/pkg/machine/libvirt/machine.go new file mode 100644 index 000000000..c38f63853 --- /dev/null +++ b/pkg/machine/libvirt/machine.go @@ -0,0 +1,15 @@ +package libvirt + +import "github.com/containers/podman/v3/pkg/machine" + +func (v *MachineVM) Init(name string, opts machine.InitOptions) error { + return nil +} + +func (v *MachineVM) Start(name string) error { + return nil +} + +func (v *MachineVM) Stop(name string) error { + return nil +} diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go new file mode 100644 index 000000000..41abe6993 --- /dev/null +++ b/pkg/machine/pull.go @@ -0,0 +1,210 @@ +package machine + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + url2 "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/containers/image/v5/pkg/compression" + "github.com/docker/docker/pkg/archive" + "github.com/sirupsen/logrus" + "github.com/vbauerster/mpb/v6" + "github.com/vbauerster/mpb/v6/decor" +) + +// GenericDownload is used when a user provides a URL +// or path for an image +type GenericDownload struct { + Download +} + +// NewGenericDownloader is used when the disk image is provided by the user +func NewGenericDownloader(vmType, vmName, pullPath string) (DistributionDownload, error) { + var ( + imageName string + ) + dataDir, err := GetDataDir(vmType) + if err != nil { + return nil, err + } + dl := Download{} + // Is pullpath a file or url? + getURL, err := url2.Parse(pullPath) + if err != nil { + return nil, err + } + if len(getURL.Scheme) > 0 { + urlSplit := strings.Split(pullPath, "/") + imageName = urlSplit[len(urlSplit)-1] + dl.LocalUncompressedFile = filepath.Join(dataDir, imageName) + dl.URL = getURL + dl.LocalPath = filepath.Join(dataDir, imageName) + } else { + // Dealing with FilePath + imageName = filepath.Base(pullPath) + dl.LocalUncompressedFile = filepath.Join(dataDir, imageName) + dl.LocalPath = pullPath + } + dl.VMName = vmName + dl.ImageName = imageName + // The download needs to be pulled into the datadir + + gd := GenericDownload{Download: dl} + gd.LocalUncompressedFile = gd.getLocalUncompressedName() + return gd, nil +} + +func (g GenericDownload) getLocalUncompressedName() string { + var ( + extension string + ) + switch { + case strings.HasSuffix(g.LocalPath, ".bz2"): + extension = ".bz2" + case strings.HasSuffix(g.LocalPath, ".gz"): + extension = ".gz" + case strings.HasSuffix(g.LocalPath, ".xz"): + extension = ".xz" + } + uncompressedFilename := filepath.Join(filepath.Dir(g.LocalUncompressedFile), g.VMName+"_"+g.ImageName) + return strings.TrimSuffix(uncompressedFilename, extension) +} + +func (g GenericDownload) DownloadImage() error { + // If we have a URL for this "downloader", we now pull it + if g.URL != nil { + if err := DownloadVMImage(g.URL, g.LocalPath); err != nil { + return err + } + } + return Decompress(g.LocalPath, g.getLocalUncompressedName()) +} + +func (g GenericDownload) Get() *Download { + return &g.Download +} + +// DownloadVMImage downloads a VM image from url to given path +// with download status +func DownloadVMImage(downloadURL fmt.Stringer, localImagePath string) error { + out, err := os.Create(localImagePath) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + logrus.Error(err) + } + }() + + resp, err := http.Get(downloadURL.String()) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error downloading VM image %s: %s", downloadURL, resp.Status) + } + size := resp.ContentLength + urlSplit := strings.Split(downloadURL.String(), "/") + prefix := "Downloading VM image: " + urlSplit[len(urlSplit)-1] + onComplete := prefix + ": done" + + p := mpb.New( + mpb.WithWidth(60), + mpb.WithRefreshRate(180*time.Millisecond), + ) + + bar := p.AddBar(size, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.OnComplete(decor.Name(prefix), onComplete), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""), + ), + ) + + proxyReader := bar.ProxyReader(resp.Body) + defer func() { + if err := proxyReader.Close(); err != nil { + logrus.Error(err) + } + }() + + if _, err := io.Copy(out, proxyReader); err != nil { + return err + } + + p.Wait() + return nil +} + +func Decompress(localPath, uncompressedPath string) error { + uncompressedFileWriter, err := os.OpenFile(uncompressedPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return err + } + sourceFile, err := ioutil.ReadFile(localPath) + if err != nil { + return err + } + + if compressionType := archive.DetectCompression(sourceFile); compressionType.Extension() == "tar.xz" { + return decompressXZ(localPath, uncompressedFileWriter) + } + return decompressEverythingElse(localPath, uncompressedFileWriter) +} + +// Will error out if file without .xz already exists +// Maybe extracting then renameing is a good idea here.. +// depends on xz: not pre-installed on mac, so it becomes a brew dependecy +func decompressXZ(src string, output io.Writer) error { + fmt.Println("Extracting compressed file") + cmd := exec.Command("xzcat", "-k", src) + //cmd := exec.Command("xz", "-d", "-k", "-v", src) + stdOut, err := cmd.StdoutPipe() + if err != nil { + return err + } + //cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + go func() { + if _, err := io.Copy(output, stdOut); err != nil { + logrus.Error(err) + } + }() + return cmd.Run() +} + +func decompressEverythingElse(src string, output io.Writer) error { + fmt.Println("Extracting compressed file") + f, err := os.Open(src) + if err != nil { + return err + } + uncompressStream, _, err := compression.AutoDecompress(f) + if err != nil { + return err + } + defer func() { + if err := uncompressStream.Close(); err != nil { + logrus.Error(err) + } + }() + + _, err = io.Copy(output, uncompressStream) + return err +} diff --git a/pkg/machine/qemu/config.go b/pkg/machine/qemu/config.go new file mode 100644 index 000000000..e4687914d --- /dev/null +++ b/pkg/machine/qemu/config.go @@ -0,0 +1,43 @@ +package qemu + +import "time" + +type MachineVM struct { + // CPUs to be assigned to the VM + CPUs uint64 + // The command line representation of the qemu command + CmdLine []string + // IdentityPath is the fq path to the ssh priv key + IdentityPath string + // IgnitionFilePath is the fq path to the .ign file + IgnitionFilePath string + // ImagePath is the fq path to + ImagePath string + // Memory in megabytes assigned to the vm + Memory uint64 + // Name of the vm + Name string + // SSH port for user networking + Port int + // QMPMonitor is the qemu monitor object for sending commands + QMPMonitor Monitor + // RemoteUsername of the vm user + RemoteUsername string +} + +type Monitor struct { + // Address portion of the qmp monitor (/tmp/tmp.sock) + Address string + // Network portion of the qmp monitor (unix) + Network string + // Timeout in seconds for qmp monitor transactions + Timeout time.Duration +} + +var ( + // defaultQMPTimeout is the timeout duration for the + // qmp monitor interactions + defaultQMPTimeout time.Duration = 2 * time.Second + // defaultRemoteUser describes the ssh username default + defaultRemoteUser = "core" +) diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go new file mode 100644 index 000000000..2652ebc10 --- /dev/null +++ b/pkg/machine/qemu/machine.go @@ -0,0 +1,512 @@ +package qemu + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/containers/podman/v3/pkg/machine" + "github.com/containers/podman/v3/utils" + "github.com/containers/storage/pkg/homedir" + "github.com/digitalocean/go-qemu/qmp" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var ( + // vmtype refers to qemu (vs libvirt, krun, etc) + vmtype = "qemu" +) + +// NewMachine initializes an instance of a virtual machine based on the qemu +// virtualization. +func NewMachine(opts machine.InitOptions) (machine.VM, error) { + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return nil, err + } + vm := new(MachineVM) + if len(opts.Name) > 0 { + vm.Name = opts.Name + } + ignitionFile := filepath.Join(vmConfigDir, vm.Name+".ign") + vm.IgnitionFilePath = ignitionFile + + // An image was specified + if len(opts.ImagePath) > 0 { + vm.ImagePath = opts.ImagePath + } + + // Assign remote user name. if not provided, use default + vm.RemoteUsername = opts.Username + if len(vm.RemoteUsername) < 1 { + vm.RemoteUsername = defaultRemoteUser + } + + // Add a random port for ssh + port, err := utils.GetRandomPort() + if err != nil { + return nil, err + } + vm.Port = port + + vm.CPUs = opts.CPUS + vm.Memory = opts.Memory + + // Look up the executable + execPath, err := exec.LookPath(QemuCommand) + if err != nil { + return nil, err + } + cmd := append([]string{execPath}) + // Add memory + cmd = append(cmd, []string{"-m", strconv.Itoa(int(vm.Memory))}...) + // Add cpus + cmd = append(cmd, []string{"-smp", strconv.Itoa(int(vm.CPUs))}...) + // Add ignition file + cmd = append(cmd, []string{"-fw_cfg", "name=opt/com.coreos/config,file=" + vm.IgnitionFilePath}...) + // Add qmp socket + monitor, err := NewQMPMonitor("unix", vm.Name, defaultQMPTimeout) + if err != nil { + return nil, err + } + vm.QMPMonitor = monitor + cmd = append(cmd, []string{"-qmp", monitor.Network + ":/" + monitor.Address + ",server=on,wait=off"}...) + + // Add network + cmd = append(cmd, "-nic", "user,model=virtio,hostfwd=tcp::"+strconv.Itoa(vm.Port)+"-:22") + + socketPath, err := getSocketDir() + if err != nil { + return nil, err + } + virtualSocketPath := filepath.Join(socketPath, "podman", vm.Name+"_ready.sock") + // Add serial port for readiness + cmd = append(cmd, []string{ + "-device", "virtio-serial", + "-chardev", "socket,path=" + virtualSocketPath + ",server=on,wait=off,id=" + vm.Name + "_ready", + "-device", "virtserialport,chardev=" + vm.Name + "_ready" + ",name=org.fedoraproject.port.0"}...) + vm.CmdLine = cmd + return vm, nil +} + +// LoadByName reads a json file that describes a known qemu vm +// and returns a vm instance +func LoadVMByName(name string) (machine.VM, error) { + vm := new(MachineVM) + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return nil, err + } + b, err := ioutil.ReadFile(filepath.Join(vmConfigDir, name+".json")) + if os.IsNotExist(err) { + return nil, errors.Wrap(machine.ErrNoSuchVM, name) + } + if err != nil { + return nil, err + } + err = json.Unmarshal(b, vm) + logrus.Debug(vm.CmdLine) + return vm, err +} + +// Init writes the json configuration file to the filesystem for +// other verbs (start, stop) +func (v *MachineVM) Init(opts machine.InitOptions) error { + var ( + key string + ) + sshDir := filepath.Join(homedir.Get(), ".ssh") + // GetConfDir creates the directory so no need to check for + // its existence + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return err + } + jsonFile := filepath.Join(vmConfigDir, v.Name) + ".json" + v.IdentityPath = filepath.Join(sshDir, v.Name) + + // The user has provided an alternate image which can be a file path + // or URL. + if len(opts.ImagePath) > 0 { + g, err := machine.NewGenericDownloader(vmtype, v.Name, opts.ImagePath) + if err != nil { + return err + } + v.ImagePath = g.Get().LocalUncompressedFile + if err := g.DownloadImage(); err != nil { + return err + } + } else { + // Get the image as usual + dd, err := machine.NewFcosDownloader(vmtype, v.Name) + if err != nil { + return err + } + v.ImagePath = dd.Get().LocalUncompressedFile + if err := dd.DownloadImage(); err != nil { + return err + } + } + + // Add arch specific options including image location + v.CmdLine = append(v.CmdLine, v.addArchOptions()...) + + // Add location of bootable image + v.CmdLine = append(v.CmdLine, "-drive", "if=virtio,file="+v.ImagePath) + // This kind of stinks but no other way around this r/n + if len(opts.IgnitionPath) < 1 { + uri := machine.SSHRemoteConnection.MakeSSHURL("localhost", "/run/user/1000/podman/podman.sock", strconv.Itoa(v.Port), v.RemoteUsername) + if err := machine.AddConnection(&uri, v.Name, filepath.Join(sshDir, v.Name), opts.IsDefault); err != nil { + return err + } + } else { + fmt.Println("An ignition path was provided. No SSH connection was added to Podman") + } + // Write the JSON file + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonFile, b, 0644); err != nil { + return err + } + + // User has provided ignition file so keygen + // will be skipped. + if len(opts.IgnitionPath) < 1 { + key, err = machine.CreateSSHKeys(v.IdentityPath) + if err != nil { + return err + } + } + // Run arch specific things that need to be done + if err := v.prepare(); err != nil { + return err + } + + originalDiskSize, err := getDiskSize(v.ImagePath) + if err != nil { + return err + } + // Resize the disk image to input disk size + // only if the virtualdisk size is less than + // the given disk size + if opts.DiskSize<<(10*3) > originalDiskSize { + resize := exec.Command("qemu-img", []string{"resize", v.ImagePath, strconv.Itoa(int(opts.DiskSize)) + "G"}...) + resize.Stdout = os.Stdout + resize.Stderr = os.Stderr + if err := resize.Run(); err != nil { + return errors.Errorf("error resizing image: %q", err) + } + } + // If the user provides an ignition file, we need to + // copy it into the conf dir + if len(opts.IgnitionPath) > 0 { + inputIgnition, err := ioutil.ReadFile(opts.IgnitionPath) + if err != nil { + return err + } + return ioutil.WriteFile(v.IgnitionFilePath, inputIgnition, 0644) + } + // Write the ignition file + ign := machine.DynamicIgnition{ + Name: opts.Username, + Key: key, + VMName: v.Name, + WritePath: v.IgnitionFilePath, + } + return machine.NewIgnitionFile(ign) +} + +// Start executes the qemu command line and forks it +func (v *MachineVM) Start(name string, _ machine.StartOptions) error { + var ( + conn net.Conn + err error + wait time.Duration = time.Millisecond * 500 + ) + attr := new(os.ProcAttr) + files := []*os.File{os.Stdin, os.Stdout, os.Stderr} + attr.Files = files + logrus.Debug(v.CmdLine) + cmd := v.CmdLine + + // Disable graphic window when not in debug mode + // Done in start, so we're not suck with the debug level we used on init + if logrus.GetLevel() != logrus.DebugLevel { + cmd = append(cmd, "-display", "none") + } + + _, err = os.StartProcess(v.CmdLine[0], cmd, attr) + if err != nil { + return err + } + fmt.Println("Waiting for VM ...") + socketPath, err := getSocketDir() + if err != nil { + return err + } + + // The socket is not made until the qemu process is running so here + // we do a backoff waiting for it. Once we have a conn, we break and + // then wait to read it. + for i := 0; i < 6; i++ { + conn, err = net.Dial("unix", filepath.Join(socketPath, "podman", v.Name+"_ready.sock")) + if err == nil { + break + } + time.Sleep(wait) + wait++ + } + if err != nil { + return err + } + _, err = bufio.NewReader(conn).ReadString('\n') + return err +} + +// Stop uses the qmp monitor to call a system_powerdown +func (v *MachineVM) Stop(name string, _ machine.StopOptions) error { + // check if the qmp socket is there. if not, qemu instance is gone + if _, err := os.Stat(v.QMPMonitor.Address); os.IsNotExist(err) { + // Right now it is NOT an error to stop a stopped machine + logrus.Debugf("QMP monitor socket %v does not exist", v.QMPMonitor.Address) + return nil + } + qmpMonitor, err := qmp.NewSocketMonitor(v.QMPMonitor.Network, v.QMPMonitor.Address, v.QMPMonitor.Timeout) + if err != nil { + return err + } + // Simple JSON formation for the QAPI + stopCommand := struct { + Execute string `json:"execute"` + }{ + Execute: "system_powerdown", + } + input, err := json.Marshal(stopCommand) + if err != nil { + return err + } + if err := qmpMonitor.Connect(); err != nil { + return err + } + defer func() { + if err := qmpMonitor.Disconnect(); err != nil { + logrus.Error(err) + } + }() + _, err = qmpMonitor.Run(input) + return err +} + +// NewQMPMonitor creates the monitor subsection of our vm +func NewQMPMonitor(network, name string, timeout time.Duration) (Monitor, error) { + rtDir, err := getSocketDir() + if err != nil { + return Monitor{}, err + } + rtDir = filepath.Join(rtDir, "podman") + if _, err := os.Stat(filepath.Join(rtDir)); os.IsNotExist(err) { + // TODO 0644 is fine on linux but macos is weird + if err := os.MkdirAll(rtDir, 0755); err != nil { + return Monitor{}, err + } + } + if timeout == 0 { + timeout = defaultQMPTimeout + } + monitor := Monitor{ + Network: network, + Address: filepath.Join(rtDir, "qmp_"+name+".sock"), + Timeout: timeout, + } + return monitor, nil +} + +func (v *MachineVM) Remove(name string, opts machine.RemoveOptions) (string, func() error, error) { + var ( + files []string + ) + + // cannot remove a running vm + if v.isRunning() { + return "", nil, errors.Errorf("running vm %q cannot be destroyed", v.Name) + } + + // Collect all the files that need to be destroyed + if !opts.SaveKeys { + files = append(files, v.IdentityPath, v.IdentityPath+".pub") + } + if !opts.SaveIgnition { + files = append(files, v.IgnitionFilePath) + } + if !opts.SaveImage { + files = append(files, v.ImagePath) + } + files = append(files, v.archRemovalFiles()...) + + if err := machine.RemoveConnection(v.Name); err != nil { + logrus.Error(err) + } + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return "", nil, err + } + files = append(files, filepath.Join(vmConfigDir, v.Name+".json")) + confirmationMessage := "\nThe following files will be deleted:\n\n" + for _, msg := range files { + confirmationMessage += msg + "\n" + } + confirmationMessage += "\n" + return confirmationMessage, func() error { + for _, f := range files { + if err := os.Remove(f); err != nil { + logrus.Error(err) + } + } + return nil + }, nil +} + +func (v *MachineVM) isRunning() bool { + // Check if qmp socket path exists + if _, err := os.Stat(v.QMPMonitor.Address); os.IsNotExist(err) { + return false + } + // Check if we can dial it + if _, err := qmp.NewSocketMonitor(v.QMPMonitor.Network, v.QMPMonitor.Address, v.QMPMonitor.Timeout); err != nil { + return false + } + return true +} + +// SSH opens an interactive SSH session to the vm specified. +// Added ssh function to VM interface: pkg/machine/config/go : line 58 +func (v *MachineVM) SSH(name string, opts machine.SSHOptions) error { + if !v.isRunning() { + return errors.Errorf("vm %q is not running.", v.Name) + } + + sshDestination := v.RemoteUsername + "@localhost" + port := strconv.Itoa(v.Port) + + args := []string{"-i", v.IdentityPath, "-p", port, sshDestination} + if len(opts.Args) > 0 { + args = append(args, opts.Args...) + } else { + fmt.Printf("Connecting to vm %s. To close connection, use `~.` or `exit`\n", v.Name) + } + + cmd := exec.Command("ssh", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + return cmd.Run() +} + +// executes qemu-image info to get the virtual disk size +// of the diskimage +func getDiskSize(path string) (uint64, error) { + diskInfo := exec.Command("qemu-img", "info", "--output", "json", path) + stdout, err := diskInfo.StdoutPipe() + if err != nil { + return 0, err + } + if err := diskInfo.Start(); err != nil { + return 0, err + } + tmpInfo := struct { + VirtualSize uint64 `json:"virtual-size"` + Filename string `json:"filename"` + ClusterSize int64 `json:"cluster-size"` + Format string `json:"format"` + FormatSpecific struct { + Type string `json:"type"` + Data map[string]string `json:"data"` + } + DirtyFlag bool `json:"dirty-flag"` + }{} + if err := json.NewDecoder(stdout).Decode(&tmpInfo); err != nil { + return 0, err + } + if err := diskInfo.Wait(); err != nil { + return 0, err + } + return tmpInfo.VirtualSize, nil +} + +// List lists all vm's that use qemu virtualization +func List(_ machine.ListOptions) ([]*machine.ListResponse, error) { + return GetVMInfos() +} + +func GetVMInfos() ([]*machine.ListResponse, error) { + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return nil, err + } + + var listed []*machine.ListResponse + + if err = filepath.Walk(vmConfigDir, func(path string, info os.FileInfo, err error) error { + vm := new(MachineVM) + if strings.HasSuffix(info.Name(), ".json") { + fullPath := filepath.Join(vmConfigDir, info.Name()) + b, err := ioutil.ReadFile(fullPath) + if err != nil { + return err + } + err = json.Unmarshal(b, vm) + if err != nil { + return err + } + listEntry := new(machine.ListResponse) + + listEntry.Name = vm.Name + listEntry.VMType = "qemu" + fi, err := os.Stat(fullPath) + if err != nil { + return err + } + listEntry.CreatedAt = fi.ModTime() + + fi, err = os.Stat(vm.ImagePath) + if err != nil { + return err + } + listEntry.LastUp = fi.ModTime() + if vm.isRunning() { + listEntry.Running = true + } + + listed = append(listed, listEntry) + } + return nil + }); err != nil { + return nil, err + } + return listed, err +} + +func IsValidVMName(name string) (bool, error) { + infos, err := GetVMInfos() + if err != nil { + return false, err + } + for _, vm := range infos { + if vm.Name == name { + return true, nil + } + } + return false, nil +} diff --git a/pkg/machine/qemu/options_darwin.go b/pkg/machine/qemu/options_darwin.go new file mode 100644 index 000000000..46ccf24cb --- /dev/null +++ b/pkg/machine/qemu/options_darwin.go @@ -0,0 +1,15 @@ +package qemu + +import ( + "os" + + "github.com/pkg/errors" +) + +func getSocketDir() (string, error) { + tmpDir, ok := os.LookupEnv("TMPDIR") + if !ok { + return "", errors.New("unable to resolve TMPDIR") + } + return tmpDir, nil +} diff --git a/pkg/machine/qemu/options_darwin_amd64.go b/pkg/machine/qemu/options_darwin_amd64.go new file mode 100644 index 000000000..69f7982b2 --- /dev/null +++ b/pkg/machine/qemu/options_darwin_amd64.go @@ -0,0 +1,18 @@ +package qemu + +var ( + QemuCommand = "qemu-system-x86_64" +) + +func (v *MachineVM) addArchOptions() []string { + opts := []string{"-cpu", "host"} + return opts +} + +func (v *MachineVM) prepare() error { + return nil +} + +func (v *MachineVM) archRemovalFiles() []string { + return []string{} +} diff --git a/pkg/machine/qemu/options_darwin_arm64.go b/pkg/machine/qemu/options_darwin_arm64.go new file mode 100644 index 000000000..7513b3048 --- /dev/null +++ b/pkg/machine/qemu/options_darwin_arm64.go @@ -0,0 +1,36 @@ +package qemu + +import ( + "os/exec" + "path/filepath" +) + +var ( + QemuCommand = "qemu-system-aarch64" +) + +func (v *MachineVM) addArchOptions() []string { + ovmfDir := getOvmfDir(v.ImagePath, v.Name) + opts := []string{ + "-accel", "hvf", + "-cpu", "cortex-a57", + "-M", "virt,highmem=off", + "-drive", "file=/usr/local/share/qemu/edk2-aarch64-code.fd,if=pflash,format=raw,readonly=on", + "-drive", "file=" + ovmfDir + ",if=pflash,format=raw"} + return opts +} + +func (v *MachineVM) prepare() error { + ovmfDir := getOvmfDir(v.ImagePath, v.Name) + cmd := []string{"dd", "if=/dev/zero", "conv=sync", "bs=1m", "count=64", "of=" + ovmfDir} + return exec.Command(cmd[0], cmd[1:]...).Run() +} + +func (v *MachineVM) archRemovalFiles() []string { + ovmDir := getOvmfDir(v.ImagePath, v.Name) + return []string{ovmDir} +} + +func getOvmfDir(imagePath, vmName string) string { + return filepath.Join(filepath.Dir(imagePath), vmName+"_ovmf_vars.fd") +} diff --git a/pkg/machine/qemu/options_linux.go b/pkg/machine/qemu/options_linux.go new file mode 100644 index 000000000..0a2e40d8f --- /dev/null +++ b/pkg/machine/qemu/options_linux.go @@ -0,0 +1,7 @@ +package qemu + +import "github.com/containers/podman/v3/pkg/util" + +func getSocketDir() (string, error) { + return util.GetRuntimeDir() +} diff --git a/pkg/machine/qemu/options_linux_amd64.go b/pkg/machine/qemu/options_linux_amd64.go new file mode 100644 index 000000000..3edd97ea1 --- /dev/null +++ b/pkg/machine/qemu/options_linux_amd64.go @@ -0,0 +1,21 @@ +package qemu + +var ( + QemuCommand = "qemu-system-x86_64" +) + +func (v *MachineVM) addArchOptions() []string { + opts := []string{ + "-accel", "kvm", + "-cpu", "host", + } + return opts +} + +func (v *MachineVM) prepare() error { + return nil +} + +func (v *MachineVM) archRemovalFiles() []string { + return []string{} +} diff --git a/pkg/machine/qemu/options_linux_arm64.go b/pkg/machine/qemu/options_linux_arm64.go new file mode 100644 index 000000000..948117653 --- /dev/null +++ b/pkg/machine/qemu/options_linux_arm64.go @@ -0,0 +1,41 @@ +package qemu + +import ( + "os" + "path/filepath" +) + +var ( + QemuCommand = "qemu-system-aarch64" +) + +func (v *MachineVM) addArchOptions() []string { + opts := []string{ + "-accel", "kvm", + "-cpu", "host", + "-M", "virt,gic-version=max", + "-bios", getQemuUefiFile("QEMU_EFI.fd"), + } + return opts +} + +func (v *MachineVM) prepare() error { + return nil +} + +func (v *MachineVM) archRemovalFiles() []string { + return []string{} +} + +func getQemuUefiFile(name string) string { + dirs := []string{ + "/usr/share/qemu-efi-aarch64", + "/usr/share/edk2/aarch64", + } + for _, dir := range dirs { + if _, err := os.Stat(dir); err == nil { + return filepath.Join(dir, name) + } + } + return name +} diff --git a/pkg/namespaces/namespaces.go b/pkg/namespaces/namespaces.go index c35f68e02..a7736aee0 100644 --- a/pkg/namespaces/namespaces.go +++ b/pkg/namespaces/namespaces.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "github.com/containers/storage" + "github.com/containers/storage/types" ) const ( @@ -109,12 +109,12 @@ func (n UsernsMode) IsDefaultValue() bool { // GetAutoOptions returns a AutoUserNsOptions with the settings to setup automatically // a user namespace. -func (n UsernsMode) GetAutoOptions() (*storage.AutoUserNsOptions, error) { +func (n UsernsMode) GetAutoOptions() (*types.AutoUserNsOptions, error) { parts := strings.SplitN(string(n), ":", 2) if parts[0] != "auto" { return nil, fmt.Errorf("wrong user namespace mode") } - options := storage.AutoUserNsOptions{} + options := types.AutoUserNsOptions{} if len(parts) == 1 { return &options, nil } @@ -131,13 +131,13 @@ func (n UsernsMode) GetAutoOptions() (*storage.AutoUserNsOptions, error) { } options.Size = uint32(s) case "uidmapping": - mapping, err := storage.ParseIDMapping([]string{v[1]}, nil, "", "") + mapping, err := types.ParseIDMapping([]string{v[1]}, nil, "", "") if err != nil { return nil, err } options.AdditionalUIDMappings = append(options.AdditionalUIDMappings, mapping.UIDMap...) case "gidmapping": - mapping, err := storage.ParseIDMapping(nil, []string{v[1]}, "", "") + mapping, err := types.ParseIDMapping(nil, []string{v[1]}, "", "") if err != nil { return nil, err } diff --git a/pkg/rootless/rootless.go b/pkg/rootless/rootless.go index b5538efc3..0b9d719a9 100644 --- a/pkg/rootless/rootless.go +++ b/pkg/rootless/rootless.go @@ -5,7 +5,7 @@ import ( "sort" "sync" - "github.com/containers/storage" + "github.com/containers/storage/pkg/lockfile" "github.com/opencontainers/runc/libcontainer/user" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -25,7 +25,7 @@ func TryJoinPauseProcess(pausePidPath string) (bool, int, error) { } // It could not join the pause process, let's lock the file before trying to delete it. - pidFileLock, err := storage.GetLockfile(pausePidPath) + pidFileLock, err := lockfile.GetLockfile(pausePidPath) if err != nil { // The file was deleted by another process. if os.IsNotExist(err) { diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index d93e4c10c..7a2bf0377 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -233,9 +233,8 @@ int is_fd_inherited(int fd) { if (open_files_set == NULL || fd > open_files_max_fd || fd < 0) - { return 0; - } + return FD_ISSET(fd % FD_SETSIZE, &(open_files_set[fd / FD_SETSIZE])) ? 1 : 0; } @@ -633,9 +632,10 @@ reexec_userns_join (int pid_to_join, char *pause_pid_file_path) close (user_ns); close (mnt_ns); - for (f = 3; f < open_files_max_fd; f++) - if (open_files_set == NULL || FD_ISSET (f % FD_SETSIZE, &(open_files_set[f / FD_SETSIZE]))) + for (f = 3; f <= open_files_max_fd; f++) + if (is_fd_inherited (f)) close (f); + return pid; } @@ -813,13 +813,14 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path, char *file_to_re if (do_socket_activation) { long num_fds; + num_fds = strtol (listen_fds, NULL, 10); if (num_fds != LONG_MIN && num_fds != LONG_MAX) { int f; for (f = 3; f < num_fds + 3; f++) - if (open_files_set == NULL || FD_ISSET (f % FD_SETSIZE, &(open_files_set[f / FD_SETSIZE]))) + if (is_fd_inherited (f)) close (f); } unsetenv ("LISTEN_PID"); diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index 03697b353..1d724ffb0 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -11,7 +11,7 @@ import ( "github.com/containers/podman/v3/libpod/image" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" - "github.com/containers/storage" + "github.com/containers/storage/types" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -367,7 +367,7 @@ func createContainerOptions(ctx context.Context, rt *libpod.Runtime, s *specgen. return options, nil } -func CreateExitCommandArgs(storageConfig storage.StoreOptions, config *config.Config, syslog, rm, exec bool) ([]string, error) { +func CreateExitCommandArgs(storageConfig types.StoreOptions, config *config.Config, syslog, rm, exec bool) ([]string, error) { // We need a cleanup process for containers in the current model. // But we can't assume that the caller is Podman - it could be another // user of the API. diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index d61c8bd19..31ed3fd7c 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -2,11 +2,13 @@ package kube import ( "context" + "encoding/json" "fmt" "net" "strings" "github.com/containers/common/pkg/parse" + "github.com/containers/common/pkg/secrets" "github.com/containers/podman/v3/libpod/image" ann "github.com/containers/podman/v3/pkg/annotations" "github.com/containers/podman/v3/pkg/specgen" @@ -94,6 +96,8 @@ type CtrSpecGenOptions struct { RestartPolicy string // NetNSIsHost tells the container to use the host netns NetNSIsHost bool + // SecretManager to access the secrets + SecretsManager *secrets.SecretsManager } func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGenerator, error) { @@ -210,12 +214,18 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener } for _, env := range opts.Container.Env { - value := envVarValue(env, opts.ConfigMaps) + value, err := envVarValue(env, opts) + if err != nil { + return nil, err + } envs[env.Name] = value } for _, envFrom := range opts.Container.EnvFrom { - cmEnvs := envVarsFromConfigMap(envFrom, opts.ConfigMaps) + cmEnvs, err := envVarsFrom(envFrom, opts) + if err != nil { + return nil, err + } for k, v := range cmEnvs { envs[k] = v @@ -325,40 +335,96 @@ func quantityToInt64(quantity *resource.Quantity) (int64, error) { return 0, errors.Errorf("Quantity cannot be represented as int64: %v", quantity) } -// envVarsFromConfigMap returns all key-value pairs as env vars from a configMap that matches the envFrom setting of a container -func envVarsFromConfigMap(envFrom v1.EnvFromSource, configMaps []v1.ConfigMap) map[string]string { +// read a k8s secret in JSON format from the secret manager +func k8sSecretFromSecretManager(name string, secretsManager *secrets.SecretsManager) (map[string][]byte, error) { + _, jsonSecret, err := secretsManager.LookupSecretData(name) + if err != nil { + return nil, err + } + + var secrets map[string][]byte + if err := json.Unmarshal(jsonSecret, &secrets); err != nil { + return nil, errors.Errorf("Secret %v is not valid JSON: %v", name, err) + } + return secrets, nil +} + +// envVarsFrom returns all key-value pairs as env vars from a configMap or secret that matches the envFrom setting of a container +func envVarsFrom(envFrom v1.EnvFromSource, opts *CtrSpecGenOptions) (map[string]string, error) { envs := map[string]string{} if envFrom.ConfigMapRef != nil { - cmName := envFrom.ConfigMapRef.Name + cmRef := envFrom.ConfigMapRef + err := errors.Errorf("Configmap %v not found", cmRef.Name) - for _, c := range configMaps { - if cmName == c.Name { + for _, c := range opts.ConfigMaps { + if cmRef.Name == c.Name { envs = c.Data + err = nil break } } + + if err != nil && (cmRef.Optional == nil || !*cmRef.Optional) { + return nil, err + } } - return envs + if envFrom.SecretRef != nil { + secRef := envFrom.SecretRef + secret, err := k8sSecretFromSecretManager(secRef.Name, opts.SecretsManager) + if err == nil { + for k, v := range secret { + envs[k] = string(v) + } + } else if secRef.Optional == nil || !*secRef.Optional { + return nil, err + } + } + + return envs, nil } // envVarValue returns the environment variable value configured within the container's env setting. -// It gets the value from a configMap if specified, otherwise returns env.Value -func envVarValue(env v1.EnvVar, configMaps []v1.ConfigMap) string { - for _, c := range configMaps { - if env.ValueFrom != nil { - if env.ValueFrom.ConfigMapKeyRef != nil { - if env.ValueFrom.ConfigMapKeyRef.Name == c.Name { - if value, ok := c.Data[env.ValueFrom.ConfigMapKeyRef.Key]; ok { - return value +// It gets the value from a configMap or secret if specified, otherwise returns env.Value +func envVarValue(env v1.EnvVar, opts *CtrSpecGenOptions) (string, error) { + if env.ValueFrom != nil { + if env.ValueFrom.ConfigMapKeyRef != nil { + cmKeyRef := env.ValueFrom.ConfigMapKeyRef + err := errors.Errorf("Cannot set env %v: configmap %v not found", env.Name, cmKeyRef.Name) + + for _, c := range opts.ConfigMaps { + if cmKeyRef.Name == c.Name { + if value, ok := c.Data[cmKeyRef.Key]; ok { + return value, nil } + err = errors.Errorf("Cannot set env %v: key %s not found in configmap %v", env.Name, cmKeyRef.Key, cmKeyRef.Name) + break + } + } + if cmKeyRef.Optional == nil || !*cmKeyRef.Optional { + return "", err + } + return "", nil + } + + if env.ValueFrom.SecretKeyRef != nil { + secKeyRef := env.ValueFrom.SecretKeyRef + secret, err := k8sSecretFromSecretManager(secKeyRef.Name, opts.SecretsManager) + if err == nil { + if val, ok := secret[secKeyRef.Key]; ok { + return string(val), nil } + err = errors.Errorf("Secret %v has not %v key", secKeyRef.Name, secKeyRef.Key) + } + if secKeyRef.Optional == nil || !*secKeyRef.Optional { + return "", errors.Errorf("Cannot set env %v: %v", env.Name, err) } + return "", nil } } - return env.Value + return env.Value, nil } // getPodPorts converts a slice of kube container descriptions to an diff --git a/pkg/specgen/generate/kube/play_test.go b/pkg/specgen/generate/kube/play_test.go index 148540e9f..f714826f0 100644 --- a/pkg/specgen/generate/kube/play_test.go +++ b/pkg/specgen/generate/kube/play_test.go @@ -1,19 +1,49 @@ package kube import ( + "encoding/json" + "io/ioutil" + "os" "testing" + "github.com/containers/common/pkg/secrets" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestEnvVarsFromConfigMap(t *testing.T) { +func createSecrets(t *testing.T, d string) *secrets.SecretsManager { + secretsManager, err := secrets.NewManager(d) + assert.NoError(t, err) + + driver := "file" + driverOpts := map[string]string{ + "path": d, + } + + for _, s := range k8sSecrets { + data, err := json.Marshal(s.Data) + assert.NoError(t, err) + + _, err = secretsManager.Store(s.ObjectMeta.Name, data, driver, driverOpts) + assert.NoError(t, err) + } + + return secretsManager +} + +func TestEnvVarsFrom(t *testing.T) { + d, err := ioutil.TempDir("", "secrets") + assert.NoError(t, err) + defer os.RemoveAll(d) + secretsManager := createSecrets(t, d) + tests := []struct { - name string - envFrom v1.EnvFromSource - configMapList []v1.ConfigMap - expected map[string]string + name string + envFrom v1.EnvFromSource + options CtrSpecGenOptions + succeed bool + expected map[string]string }{ { "ConfigMapExists", @@ -24,7 +54,10 @@ func TestEnvVarsFromConfigMap(t *testing.T) { }, }, }, - configMapList, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + true, map[string]string{ "myvar": "foo", }, @@ -38,7 +71,26 @@ func TestEnvVarsFromConfigMap(t *testing.T) { }, }, }, - configMapList, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + false, + nil, + }, + { + "OptionalConfigMapDoesNotExist", + v1.EnvFromSource{ + ConfigMapRef: &v1.ConfigMapEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + Optional: &optional, + }, + }, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + true, map[string]string{}, }, { @@ -50,7 +102,74 @@ func TestEnvVarsFromConfigMap(t *testing.T) { }, }, }, - []v1.ConfigMap{}, + CtrSpecGenOptions{ + ConfigMaps: []v1.ConfigMap{}, + }, + false, + nil, + }, + { + "OptionalEmptyConfigMapList", + v1.EnvFromSource{ + ConfigMapRef: &v1.ConfigMapEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Optional: &optional, + }, + }, + CtrSpecGenOptions{ + ConfigMaps: []v1.ConfigMap{}, + }, + true, + map[string]string{}, + }, + { + "SecretExists", + v1.EnvFromSource{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + true, + map[string]string{ + "myvar": "foo", + }, + }, + { + "SecretDoesNotExist", + v1.EnvFromSource{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + false, + nil, + }, + { + "OptionalSecretDoesNotExist", + v1.EnvFromSource{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + Optional: &optional, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + true, map[string]string{}, }, } @@ -58,18 +177,25 @@ func TestEnvVarsFromConfigMap(t *testing.T) { for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { - result := envVarsFromConfigMap(test.envFrom, test.configMapList) + result, err := envVarsFrom(test.envFrom, &test.options) + assert.Equal(t, err == nil, test.succeed) assert.Equal(t, test.expected, result) }) } } func TestEnvVarValue(t *testing.T) { + d, err := ioutil.TempDir("", "secrets") + assert.NoError(t, err) + defer os.RemoveAll(d) + secretsManager := createSecrets(t, d) + tests := []struct { - name string - envVar v1.EnvVar - configMapList []v1.ConfigMap - expected string + name string + envVar v1.EnvVar + options CtrSpecGenOptions + succeed bool + expected string }{ { "ConfigMapExists", @@ -84,7 +210,10 @@ func TestEnvVarValue(t *testing.T) { }, }, }, - configMapList, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + true, "foo", }, { @@ -100,7 +229,30 @@ func TestEnvVarValue(t *testing.T) { }, }, }, - configMapList, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + false, + "", + }, + { + "OptionalContainerKeyDoesNotExistInConfigMap", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + ConfigMapKeyRef: &v1.ConfigMapKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Key: "doesnotexist", + Optional: &optional, + }, + }, + }, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + true, "", }, { @@ -116,7 +268,30 @@ func TestEnvVarValue(t *testing.T) { }, }, }, - configMapList, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + false, + "", + }, + { + "OptionalConfigMapDoesNotExist", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + ConfigMapKeyRef: &v1.ConfigMapKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + Key: "myvar", + Optional: &optional, + }, + }, + }, + CtrSpecGenOptions{ + ConfigMaps: configMapList, + }, + true, "", }, { @@ -132,7 +307,127 @@ func TestEnvVarValue(t *testing.T) { }, }, }, - []v1.ConfigMap{}, + CtrSpecGenOptions{ + ConfigMaps: []v1.ConfigMap{}, + }, + false, + "", + }, + { + "OptionalEmptyConfigMapList", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + ConfigMapKeyRef: &v1.ConfigMapKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Key: "myvar", + Optional: &optional, + }, + }, + }, + CtrSpecGenOptions{ + ConfigMaps: []v1.ConfigMap{}, + }, + true, + "", + }, + { + "SecretExists", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Key: "myvar", + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + true, + "foo", + }, + { + "ContainerKeyDoesNotExistInSecret", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Key: "doesnotexist", + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + false, + "", + }, + { + "OptionalContainerKeyDoesNotExistInSecret", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "foo", + }, + Key: "doesnotexist", + Optional: &optional, + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + true, + "", + }, + { + "SecretDoesNotExist", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + Key: "myvar", + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + false, + "", + }, + { + "OptionalSecretDoesNotExist", + v1.EnvVar{ + Name: "FOO", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "doesnotexist", + }, + Key: "myvar", + Optional: &optional, + }, + }, + }, + CtrSpecGenOptions{ + SecretsManager: secretsManager, + }, + true, "", }, } @@ -140,7 +435,8 @@ func TestEnvVarValue(t *testing.T) { for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { - result := envVarValue(test.envVar, test.configMapList) + result, err := envVarValue(test.envVar, &test.options) + assert.Equal(t, err == nil, test.succeed) assert.Equal(t, test.expected, result) }) } @@ -170,3 +466,30 @@ var configMapList = []v1.ConfigMap{ }, }, } + +var optional = true + +var k8sSecrets = []v1.Secret{ + { + TypeMeta: v12.TypeMeta{ + Kind: "Secret", + }, + ObjectMeta: v12.ObjectMeta{ + Name: "bar", + }, + Data: map[string][]byte{ + "myvar": []byte("bar"), + }, + }, + { + TypeMeta: v12.TypeMeta{ + Kind: "Secret", + }, + ObjectMeta: v12.ObjectMeta{ + Name: "foo", + }, + Data: map[string][]byte{ + "myvar": []byte("foo"), + }, + }, +} diff --git a/pkg/specgen/generate/kube/volume.go b/pkg/specgen/generate/kube/volume.go index e4f3eb196..a8042b532 100644 --- a/pkg/specgen/generate/kube/volume.go +++ b/pkg/specgen/generate/kube/volume.go @@ -116,7 +116,7 @@ func InitializeVolumes(specVolumes []v1.Volume) (map[string]*KubeVolume, error) for _, specVolume := range specVolumes { volume, err := VolumeFromSource(specVolume.VolumeSource) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "failed to create volume %q", specVolume.Name) } volumes[specVolume.Name] = volume diff --git a/pkg/specgen/generate/ports.go b/pkg/specgen/generate/ports.go index 6cf83ed81..678e36a70 100644 --- a/pkg/specgen/generate/ports.go +++ b/pkg/specgen/generate/ports.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" + "github.com/containers/podman/v3/utils" + "github.com/containers/podman/v3/libpod/image" "github.com/containers/podman/v3/pkg/specgen" "github.com/cri-o/ocicni/pkg/ocicni" @@ -218,7 +220,7 @@ func parsePortMapping(portMappings []specgen.PortMapping) ([]ocicni.PortMapping, // Only get a random candidate for single entries or the start // of a range. Otherwise we just increment the candidate. if !tmp.isInRange || tmp.startOfRange { - candidate, err = getRandomPort() + candidate, err = utils.GetRandomPort() if err != nil { return nil, nil, nil, errors.Wrapf(err, "error getting candidate host port for container port %d", p.ContainerPort) } @@ -344,7 +346,7 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, img *imag for hostPort == 0 && tries > 0 { // We can't select a specific protocol, which is // unfortunate for the UDP case. - candidate, err := getRandomPort() + candidate, err := utils.GetRandomPort() if err != nil { return nil, err } @@ -419,21 +421,3 @@ func checkProtocol(protocol string, allowSCTP bool) ([]string, error) { return finalProto, nil } - -// Find a random, open port on the host -func getRandomPort() (int, error) { - l, err := net.Listen("tcp", ":0") - if err != nil { - return 0, errors.Wrapf(err, "unable to get free TCP port") - } - defer l.Close() - _, randomPort, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - return 0, errors.Wrapf(err, "unable to determine free port") - } - rp, err := strconv.Atoi(randomPort) - if err != nil { - return 0, errors.Wrapf(err, "unable to convert random port to int") - } - return rp, nil -} diff --git a/pkg/specgen/namespaces.go b/pkg/specgen/namespaces.go index fb7d65da4..f665fc0be 100644 --- a/pkg/specgen/namespaces.go +++ b/pkg/specgen/namespaces.go @@ -54,7 +54,7 @@ const ( // Namespace describes the namespace type Namespace struct { NSMode NamespaceMode `json:"nsmode,omitempty"` - Value string `json:"string,omitempty"` + Value string `json:"value,omitempty"` } // IsDefault returns whether the namespace is set to the default setting (which diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index 732579bf0..c10dc5ef5 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -5,7 +5,7 @@ import ( "syscall" "github.com/containers/image/v5/manifest" - "github.com/containers/storage" + "github.com/containers/storage/types" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -302,7 +302,7 @@ type ContainerSecurityConfig struct { // IDMappings are UID and GID mappings that will be used by user // namespaces. // Required if UserNS is private. - IDMappings *storage.IDMappingOptions `json:"idmappings,omitempty"` + IDMappings *types.IDMappingOptions `json:"idmappings,omitempty"` // ReadOnlyFilesystem indicates that everything will be mounted // as read-only ReadOnlyFilesystem bool `json:"read_only_filesystem,omitempty"` diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go index 94a6f4cb5..eafd45528 100644 --- a/pkg/systemd/generate/common.go +++ b/pkg/systemd/generate/common.go @@ -36,22 +36,49 @@ Description=Podman {{{{.ServiceName}}}}.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor={{{{.GraphRoot}}}} {{{{.RunRoot}}}} ` -// filterPodFlags removes --pod and --pod-id-file from the specified command. -func filterPodFlags(command []string) []string { +// filterPodFlags removes --pod, --pod-id-file and --infra-conmon-pidfile from the specified command. +// argCount is the number of last arguments which should not be filtered, e.g. the container entrypoint. +func filterPodFlags(command []string, argCount int) []string { processed := []string{} - for i := 0; i < len(command); i++ { + for i := 0; i < len(command)-argCount; i++ { s := command[i] - if s == "--pod" || s == "--pod-id-file" { + if s == "--pod" || s == "--pod-id-file" || s == "--infra-conmon-pidfile" { i++ continue } - if strings.HasPrefix(s, "--pod=") || strings.HasPrefix(s, "--pod-id-file=") { + if strings.HasPrefix(s, "--pod=") || + strings.HasPrefix(s, "--pod-id-file=") || + strings.HasPrefix(s, "--infra-conmon-pidfile=") { continue } processed = append(processed, s) } + processed = append(processed, command[len(command)-argCount:]...) + return processed +} + +// filterCommonContainerFlags removes --conmon-pidfile, --cidfile and --cgroups from the specified command. +// argCount is the number of last arguments which should not be filtered, e.g. the container entrypoint. +func filterCommonContainerFlags(command []string, argCount int) []string { + processed := []string{} + for i := 0; i < len(command)-argCount; i++ { + s := command[i] + + switch { + case s == "--conmon-pidfile", s == "--cidfile", s == "--cgroups": + i++ + continue + case strings.HasPrefix(s, "--conmon-pidfile="), + strings.HasPrefix(s, "--cidfile="), + strings.HasPrefix(s, "--cgroups="): + continue + } + processed = append(processed, s) + } + processed = append(processed, command[len(command)-argCount:]...) return processed } diff --git a/pkg/systemd/generate/common_test.go b/pkg/systemd/generate/common_test.go index 3787e461e..30e758127 100644 --- a/pkg/systemd/generate/common_test.go +++ b/pkg/systemd/generate/common_test.go @@ -1,7 +1,6 @@ package generate import ( - "strings" "testing" "github.com/stretchr/testify/assert" @@ -9,22 +8,144 @@ import ( func TestFilterPodFlags(t *testing.T) { tests := []struct { - input []string + input []string + output []string + argCount int }{ - {[]string{"podman", "pod", "create"}}, - {[]string{"podman", "pod", "create", "--name", "foo"}}, - {[]string{"podman", "pod", "create", "--pod-id-file", "foo"}}, - {[]string{"podman", "pod", "create", "--pod-id-file=foo"}}, - {[]string{"podman", "run", "--pod", "foo"}}, - {[]string{"podman", "run", "--pod=foo"}}, + { + []string{"podman", "pod", "create"}, + []string{"podman", "pod", "create"}, + 0, + }, + { + []string{"podman", "pod", "create", "--name", "foo"}, + []string{"podman", "pod", "create", "--name", "foo"}, + 0, + }, + { + []string{"podman", "pod", "create", "--pod-id-file", "foo"}, + []string{"podman", "pod", "create"}, + 0, + }, + { + []string{"podman", "pod", "create", "--pod-id-file=foo"}, + []string{"podman", "pod", "create"}, + 0, + }, + { + []string{"podman", "pod", "create", "--pod-id-file", "foo", "--infra-conmon-pidfile", "foo"}, + []string{"podman", "pod", "create"}, + 0, + }, + { + []string{"podman", "pod", "create", "--pod-id-file", "foo", "--infra-conmon-pidfile=foo"}, + []string{"podman", "pod", "create"}, + 0, + }, + { + []string{"podman", "run", "--pod", "foo"}, + []string{"podman", "run"}, + 0, + }, + { + []string{"podman", "run", "--pod=foo"}, + []string{"podman", "run"}, + 0, + }, + { + []string{"podman", "run", "--pod=foo", "fedora", "podman", "run", "--pod=test", "alpine"}, + []string{"podman", "run", "fedora", "podman", "run", "--pod=test", "alpine"}, + 5, + }, + { + []string{"podman", "run", "--pod", "foo", "fedora", "podman", "run", "--pod", "test", "alpine"}, + []string{"podman", "run", "fedora", "podman", "run", "--pod", "test", "alpine"}, + 6, + }, + { + []string{"podman", "run", "--pod-id-file=foo", "fedora", "podman", "run", "--pod-id-file=test", "alpine"}, + []string{"podman", "run", "fedora", "podman", "run", "--pod-id-file=test", "alpine"}, + 5, + }, + { + []string{"podman", "run", "--pod-id-file", "foo", "fedora", "podman", "run", "--pod-id-file", "test", "alpine"}, + []string{"podman", "run", "fedora", "podman", "run", "--pod-id-file", "test", "alpine"}, + 6, + }, + } + + for _, test := range tests { + processed := filterPodFlags(test.input, test.argCount) + assert.Equal(t, test.output, processed) + } +} + +func TestFilterCommonContainerFlags(t *testing.T) { + tests := []struct { + input []string + output []string + argCount int + }{ + { + []string{"podman", "run", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--conmon-pidfile", "foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--conmon-pidfile=foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cidfile", "foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cidfile=foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cgroups", "foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cgroups=foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cgroups", "foo", "--conmon-pidfile", "foo", "--cidfile", "foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "alpine"}, + []string{"podman", "run", "alpine"}, + 1, + }, + { + []string{"podman", "run", "--cgroups", "foo", "--conmon-pidfile", "foo", "--cidfile", "foo", "alpine", "--cgroups", "foo", "--conmon-pidfile", "foo", "--cidfile", "foo"}, + []string{"podman", "run", "alpine", "--cgroups", "foo", "--conmon-pidfile", "foo", "--cidfile", "foo"}, + 7, + }, + { + []string{"podman", "run", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "alpine", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo"}, + []string{"podman", "run", "alpine", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo"}, + 4, + }, } for _, test := range tests { - processed := filterPodFlags(test.input) - for _, s := range processed { - assert.False(t, strings.HasPrefix(s, "--pod-id-file")) - assert.False(t, strings.HasPrefix(s, "--pod")) - } + processed := filterCommonContainerFlags(test.input, test.argCount) + assert.Equal(t, test.output, processed) } } diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go index 9343a5067..e06655a8d 100644 --- a/pkg/systemd/generate/containers.go +++ b/pkg/systemd/generate/containers.go @@ -71,6 +71,12 @@ type containerInfo struct { // If not nil, the container is part of the pod. We can use the // podInfo to extract the relevant data. Pod *podInfo + // Location of the GraphRoot for the container. Required for ensuring the + // volume has finished mounting when coming online at boot. + GraphRoot string + // Location of the RunRoot for the container. Required for ensuring the tmpfs + // or volume exists and is mounted when coming online at boot. + RunRoot string } const containerTemplate = headerTemplate + ` @@ -132,6 +138,21 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste nameOrID, serviceName := containerServiceName(ctr, options) + store := ctr.Runtime().GetStore() + if store == nil { + return nil, errors.Errorf("could not determine storage store for container") + } + + graphRoot := store.GraphRoot() + if graphRoot == "" { + return nil, errors.Errorf("could not lookup container's graphroot: got empty string") + } + + runRoot := store.RunRoot() + if runRoot == "" { + return nil, errors.Errorf("could not lookup container's runroot: got empty string") + } + info := containerInfo{ ServiceName: serviceName, ContainerNameOrID: nameOrID, @@ -140,6 +161,8 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste StopTimeout: timeout, GenerateTimestamp: true, CreateCommand: createCommand, + GraphRoot: graphRoot, + RunRoot: runRoot, } return &info, nil @@ -215,13 +238,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst "--cidfile", "{{{{.ContainerIDFile}}}}", "--cgroups=no-conmon", ) - // If the container is in a pod, make sure that the - // --pod-id-file is set correctly. - if info.Pod != nil { - podFlags := []string{"--pod-id-file", "{{{{.Pod.PodIDFile}}}}"} - startCommand = append(startCommand, podFlags...) - info.CreateCommand = filterPodFlags(info.CreateCommand) - } + remainingCmd := info.CreateCommand[index:] // Presence check for certain flags/options. fs := pflag.NewFlagSet("args", pflag.ContinueOnError) @@ -231,7 +248,16 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst fs.BoolP("detach", "d", false, "") fs.String("name", "", "") fs.Bool("replace", false, "") - fs.Parse(info.CreateCommand[index:]) + fs.Parse(remainingCmd) + + remainingCmd = filterCommonContainerFlags(remainingCmd, fs.NArg()) + // If the container is in a pod, make sure that the + // --pod-id-file is set correctly. + if info.Pod != nil { + podFlags := []string{"--pod-id-file", "{{{{.Pod.PodIDFile}}}}"} + startCommand = append(startCommand, podFlags...) + remainingCmd = filterPodFlags(remainingCmd, fs.NArg()) + } hasDetachParam, err := fs.GetBool("detach") if err != nil { @@ -243,8 +269,6 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst return "", err } - remainingCmd := info.CreateCommand[index:] - if !hasDetachParam { // Enforce detaching // diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go index ebbbdb786..899ba6bfa 100644 --- a/pkg/systemd/generate/containers_test.go +++ b/pkg/systemd/generate/containers_test.go @@ -48,6 +48,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -73,6 +74,7 @@ Description=Podman container-foobar.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -96,6 +98,7 @@ Description=Podman container-foobar.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage BindsTo=a.service b.service c.service pod.service After=a.service b.service c.service pod.service @@ -121,6 +124,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -145,6 +149,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -169,6 +174,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -193,6 +199,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -217,6 +224,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -242,6 +250,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -270,6 +279,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -294,6 +304,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -318,6 +329,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -342,6 +354,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -366,6 +379,7 @@ Description=Podman jadda-jadda.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage [Service] Environment=PODMAN_SYSTEMD_UNIT=%n @@ -381,6 +395,56 @@ Type=forking [Install] WantedBy=multi-user.target default.target ` + + goodNewWithIDFiles := `# jadda-jadda.service +# autogenerated by Podman CI + +[Unit] +Description=Podman jadda-jadda.service +Documentation=man:podman-generate-systemd(1) +Wants=network.target +After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=always +TimeoutStopSec=70 +ExecStartPre=/bin/rm -f %t/jadda-jadda.pid %t/jadda-jadda.ctr-id +ExecStart=/usr/bin/podman run --conmon-pidfile %t/jadda-jadda.pid --cidfile %t/jadda-jadda.ctr-id --cgroups=no-conmon -d awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/jadda-jadda.ctr-id -t 10 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/jadda-jadda.ctr-id +PIDFile=%t/jadda-jadda.pid +Type=forking + +[Install] +WantedBy=multi-user.target default.target +` + + goodNewWithPodIDFiles := `# jadda-jadda.service +# autogenerated by Podman CI + +[Unit] +Description=Podman jadda-jadda.service +Documentation=man:podman-generate-systemd(1) +Wants=network.target +After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=always +TimeoutStopSec=70 +ExecStartPre=/bin/rm -f %t/jadda-jadda.pid %t/jadda-jadda.ctr-id +ExecStart=/usr/bin/podman run --conmon-pidfile %t/jadda-jadda.pid --cidfile %t/jadda-jadda.ctr-id --cgroups=no-conmon --pod-id-file %t/pod-foobar.pod-id-file -d awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine +ExecStop=/usr/bin/podman stop --ignore --cidfile %t/jadda-jadda.ctr-id -t 10 +ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/jadda-jadda.ctr-id +PIDFile=%t/jadda-jadda.pid +Type=forking + +[Install] +WantedBy=multi-user.target default.target +` tests := []struct { name string info containerInfo @@ -400,6 +464,8 @@ WantedBy=multi-user.target default.target StopTimeout: 22, PodmanVersion: "CI", EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodID, false, @@ -416,6 +482,8 @@ WantedBy=multi-user.target default.target StopTimeout: 22, PodmanVersion: "CI", EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodIDNoHeaderInfo, false, @@ -432,6 +500,8 @@ WantedBy=multi-user.target default.target StopTimeout: 10, PodmanVersion: "CI", EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodName, false, @@ -449,6 +519,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", BoundToServices: []string{"pod", "a", "b", "c"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNameBoundTo, false, @@ -464,6 +536,8 @@ WantedBy=multi-user.target default.target StopTimeout: 10, PodmanVersion: "CI", EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, "", false, @@ -481,6 +555,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "container", "run", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodWithNameAndGeneric, true, @@ -498,6 +574,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodWithExplicitShortDetachParam, true, @@ -515,6 +593,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", Pod: &podInfo{ PodIDFile: "%t/pod-foobar.pod-id-file", }, @@ -535,6 +615,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "--detach", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNameNewDetach, true, @@ -552,6 +634,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodIDNew, true, @@ -569,6 +653,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "--detach=true", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, genGoodNewDetach("--detach=true"), true, @@ -586,6 +672,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "--detach=false", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, genGoodNewDetach("-d"), true, @@ -603,6 +691,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNameNewDetachFalseWithCmd, true, @@ -620,6 +710,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNameNewDetachFalseWithCmd, true, @@ -637,6 +729,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "-dti", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, genGoodNewDetach("-dti"), true, @@ -654,6 +748,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "run", "-tid", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, genGoodNewDetach("-tid"), true, @@ -671,6 +767,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "--events-backend", "none", "--runroot", "/root", "run", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNewRootFlags, true, @@ -688,6 +786,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "container", "create", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodContainerCreate, true, @@ -705,6 +805,8 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "create", "--name", "test", "--log-driver=journald", "--log-opt=tag={{.Name}}", "awesome-image:latest"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNewWithJournaldTag, true, @@ -722,12 +824,55 @@ WantedBy=multi-user.target default.target PodmanVersion: "CI", CreateCommand: []string{"I'll get stripped", "create", "--name", "test", "awesome-image:latest", "sh", "-c", "kill $$ && echo %\\"}, EnvVariable: define.EnvVariable, + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", }, goodNewWithSpecialChars, true, false, false, }, + {"good with ID files", + containerInfo{ + Executable: "/usr/bin/podman", + ServiceName: "jadda-jadda", + ContainerNameOrID: "jadda-jadda", + RestartPolicy: "always", + PIDFile: "/var/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", + StopTimeout: 10, + PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", + CreateCommand: []string{"I'll get stripped", "create", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "awesome-image:latest", "podman", "run", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "alpine"}, + EnvVariable: define.EnvVariable, + }, + goodNewWithIDFiles, + true, + false, + false, + }, + {"good with pod ID files", + containerInfo{ + Executable: "/usr/bin/podman", + ServiceName: "jadda-jadda", + ContainerNameOrID: "jadda-jadda", + RestartPolicy: "always", + PIDFile: "/var/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", + StopTimeout: 10, + PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", + CreateCommand: []string{"I'll get stripped", "create", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "--pod", "test", "awesome-image:latest", "podman", "run", "--cgroups=foo", "--conmon-pidfile=foo", "--cidfile=foo", "--pod-id-file", "/tmp/pod-foobar.pod-id-file", "alpine"}, + EnvVariable: define.EnvVariable, + Pod: &podInfo{ + PodIDFile: "%t/pod-foobar.pod-id-file", + }, + }, + goodNewWithPodIDFiles, + true, + false, + false, + }, } for _, tt := range tests { test := tt diff --git a/pkg/systemd/generate/pods.go b/pkg/systemd/generate/pods.go index f96058d36..1b92649e8 100644 --- a/pkg/systemd/generate/pods.go +++ b/pkg/systemd/generate/pods.go @@ -73,6 +73,12 @@ type podInfo struct { ExecStopPost string // Removes autogenerated by Podman and timestamp if set to true GenerateNoHeader bool + // Location of the GraphRoot for the pod. Required for ensuring the + // volume has finished mounting when coming online at boot. + GraphRoot string + // Location of the RunRoot for the pod. Required for ensuring the tmpfs + // or volume exists and is mounted when coming online at boot. + RunRoot string } const podTemplate = headerTemplate + `Requires={{{{- range $index, $value := .RequiredServices -}}}}{{{{if $index}}}} {{{{end}}}}{{{{ $value }}}}.service{{{{end}}}} @@ -273,16 +279,16 @@ func executePodTemplate(info *podInfo, options entities.GenerateSystemdOptions) } podRootArgs = info.CreateCommand[1 : podCreateIndex-1] info.RootFlags = strings.Join(escapeSystemdArguments(podRootArgs), " ") - podCreateArgs = filterPodFlags(info.CreateCommand[podCreateIndex+1:]) + podCreateArgs = filterPodFlags(info.CreateCommand[podCreateIndex+1:], 0) } // We're hard-coding the first five arguments and append the // CreateCommand with a stripped command and subcommand. startCommand := []string{info.Executable} startCommand = append(startCommand, podRootArgs...) startCommand = append(startCommand, - []string{"pod", "create", - "--infra-conmon-pidfile", "{{{{.PIDFile}}}}", - "--pod-id-file", "{{{{.PodIDFile}}}}"}...) + "pod", "create", + "--infra-conmon-pidfile", "{{{{.PIDFile}}}}", + "--pod-id-file", "{{{{.PodIDFile}}}}") // Presence check for certain flags/options. fs := pflag.NewFlagSet("args", pflag.ContinueOnError) diff --git a/pkg/systemd/generate/pods_test.go b/pkg/systemd/generate/pods_test.go index 50c8d4556..0e4d92c50 100644 --- a/pkg/systemd/generate/pods_test.go +++ b/pkg/systemd/generate/pods_test.go @@ -47,6 +47,7 @@ Description=Podman pod-123abc.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage Requires=container-1.service container-2.service Before=container-1.service container-2.service @@ -74,6 +75,7 @@ Description=Podman pod-123abc.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage Requires=container-1.service container-2.service Before=container-1.service container-2.service @@ -101,6 +103,7 @@ Description=Podman pod-123abc.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage Requires=container-1.service container-2.service Before=container-1.service container-2.service @@ -128,6 +131,7 @@ Description=Podman pod-123abc.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage Requires=container-1.service container-2.service Before=container-1.service container-2.service @@ -155,6 +159,7 @@ Description=Podman pod-123abc.service Documentation=man:podman-generate-systemd(1) Wants=network.target After=network-online.target +RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage Requires=container-1.service container-2.service Before=container-1.service container-2.service @@ -191,6 +196,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 42, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"}, }, @@ -208,6 +215,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 42, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"}, }, @@ -225,6 +234,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 42, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"}, }, @@ -242,6 +253,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 10, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"}, }, @@ -259,6 +272,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 10, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"}, }, @@ -276,6 +291,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 10, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "pod", "create", "--name", "foo", "--replace=false"}, }, @@ -293,6 +310,8 @@ WantedBy=multi-user.target default.target PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", StopTimeout: 10, PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", RequiredServices: []string{"container-1", "container-2"}, CreateCommand: []string{"podman", "pod", "create", "--name", "foo", "--label", "key={{someval}}"}, }, @@ -301,6 +320,25 @@ WantedBy=multi-user.target default.target false, false, }, + {"pod --new with ID files", + podInfo{ + Executable: "/usr/bin/podman", + ServiceName: "pod-123abc", + InfraNameOrID: "jadda-jadda-infra", + RestartPolicy: "on-failure", + PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid", + StopTimeout: 10, + PodmanVersion: "CI", + GraphRoot: "/var/lib/containers/storage", + RunRoot: "/var/run/containers/storage", + RequiredServices: []string{"container-1", "container-2"}, + CreateCommand: []string{"podman", "pod", "create", "--infra-conmon-pidfile", "/tmp/pod-123abc.pid", "--pod-id-file", "/tmp/pod-123abc.pod-id", "--name", "foo", "bar=arg with space"}, + }, + podGoodNamedNew, + true, + false, + false, + }, } for _, tt := range tests { diff --git a/pkg/util/filters.go b/pkg/util/filters.go index 51b2c5331..43bf646f1 100644 --- a/pkg/util/filters.go +++ b/pkg/util/filters.go @@ -11,11 +11,11 @@ import ( "github.com/pkg/errors" ) -// ComputeUntilTimestamp extracts unitil timestamp from filters -func ComputeUntilTimestamp(filter string, filterValues []string) (time.Time, error) { +// ComputeUntilTimestamp extracts until timestamp from filters +func ComputeUntilTimestamp(filterValues []string) (time.Time, error) { invalid := time.Time{} if len(filterValues) != 1 { - return invalid, errors.Errorf("specify exactly one timestamp for %s", filter) + return invalid, errors.Errorf("specify exactly one timestamp for until") } ts, err := timetype.GetTimestamp(filterValues[0], time.Now()) if err != nil { @@ -93,3 +93,24 @@ func PrepareFilters(r *http.Request) (*map[string][]string, error) { } return &filterMap, nil } + +// MatchLabelFilters matches labels and returs true if they are valid +func MatchLabelFilters(filterValues []string, labels map[string]string) bool { +outer: + for _, filterValue := range filterValues { + filterArray := strings.SplitN(filterValue, "=", 2) + filterKey := filterArray[0] + if len(filterArray) > 1 { + filterValue = filterArray[1] + } else { + filterValue = "" + } + for labelKey, labelValue := range labels { + if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { + continue outer + } + } + return false + } + return true +} diff --git a/pkg/util/filters_test.go b/pkg/util/filters_test.go new file mode 100644 index 000000000..47259013e --- /dev/null +++ b/pkg/util/filters_test.go @@ -0,0 +1,113 @@ +package util + +import ( + "testing" +) + +func TestMatchLabelFilters(t *testing.T) { + testLabels := map[string]string{ + "label1": "", + "label2": "test", + "label3": "", + } + type args struct { + filterValues []string + labels map[string]string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Match when all filters the same as labels", + args: args{ + filterValues: []string{"label1", "label3", "label2=test"}, + labels: testLabels, + }, + want: true, + }, + { + name: "Match when filter value not provided in args", + args: args{ + filterValues: []string{"label2"}, + labels: testLabels, + }, + want: true, + }, + { + name: "Match when no filter value is given", + args: args{ + filterValues: []string{"label2="}, + labels: testLabels, + }, + want: true, + }, + { + name: "Do not match when filter value differs", + args: args{ + filterValues: []string{"label2=differs"}, + labels: testLabels, + }, + want: false, + }, + { + name: "Do not match when filter value not listed in labels", + args: args{ + filterValues: []string{"label1=xyz"}, + labels: testLabels, + }, + want: false, + }, + { + name: "Do not match when one from many not ok", + args: args{ + filterValues: []string{"label1=xyz", "invalid=valid"}, + labels: testLabels, + }, + want: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := MatchLabelFilters(tt.args.filterValues, tt.args.labels); got != tt.want { + t.Errorf("MatchLabelFilters() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComputeUntilTimestamp(t *testing.T) { + tests := []struct { + name string + args []string + wantErr bool + }{ + { + name: "Return error when more values in list", + args: []string{"5h", "6s"}, + wantErr: true, + }, + { + name: "Return error when invalid time", + args: []string{"invalidTime"}, + wantErr: true, + }, + { + name: "Do not return error when correct time format supplied", + args: []string{"44m"}, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + _, err := ComputeUntilTimestamp(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("ComputeUntilTimestamp() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index a4c8f3a64..bbaf72981 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -20,8 +20,8 @@ import ( "github.com/containers/podman/v3/pkg/namespaces" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/signal" - "github.com/containers/storage" "github.com/containers/storage/pkg/idtools" + stypes "github.com/containers/storage/types" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -344,8 +344,8 @@ func ParseSignal(rawSignal string) (syscall.Signal, error) { } // GetKeepIDMapping returns the mappings and the user to use when keep-id is used -func GetKeepIDMapping() (*storage.IDMappingOptions, int, int, error) { - options := storage.IDMappingOptions{ +func GetKeepIDMapping() (*stypes.IDMappingOptions, int, int, error) { + options := stypes.IDMappingOptions{ HostUIDMapping: true, HostGIDMapping: true, } @@ -395,8 +395,8 @@ func GetKeepIDMapping() (*storage.IDMappingOptions, int, int, error) { } // ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping -func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*storage.IDMappingOptions, error) { - options := storage.IDMappingOptions{ +func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*stypes.IDMappingOptions, error) { + options := stypes.IDMappingOptions{ HostUIDMapping: true, HostGIDMapping: true, } @@ -479,7 +479,7 @@ type tomlConfig struct { } `toml:"storage"` } -func getTomlStorage(storeOptions *storage.StoreOptions) *tomlConfig { +func getTomlStorage(storeOptions *stypes.StoreOptions) *tomlConfig { config := new(tomlConfig) config.Storage.Driver = storeOptions.GraphDriverName @@ -496,7 +496,7 @@ func getTomlStorage(storeOptions *storage.StoreOptions) *tomlConfig { } // WriteStorageConfigFile writes the configuration to a file -func WriteStorageConfigFile(storageOpts *storage.StoreOptions, storageConf string) error { +func WriteStorageConfigFile(storageOpts *stypes.StoreOptions, storageConf string) error { if err := os.MkdirAll(filepath.Dir(storageConf), 0755); err != nil { return err } |