diff options
Diffstat (limited to 'pkg')
47 files changed, 2314 insertions, 351 deletions
diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index d3277b815..e7146a5d8 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -19,6 +19,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/containers/podman/v3/pkg/ps" "github.com/containers/podman/v3/pkg/signal" + "github.com/containers/podman/v3/pkg/util" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" @@ -92,23 +93,24 @@ func ListContainers(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) query := struct { - All bool `schema:"all"` - Limit int `schema:"limit"` - Size bool `schema:"size"` - Filters map[string][]string `schema:"filters"` + All bool `schema:"all"` + Limit int `schema:"limit"` + Size bool `schema:"size"` }{ // override any golang type defaults } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + filterMap, err := util.PrepareFilters(r) + + if dErr := decoder.Decode(&query, r.URL.Query()); dErr != nil || err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - filterFuncs := make([]libpod.ContainerFilter, 0, len(query.Filters)) + filterFuncs := make([]libpod.ContainerFilter, 0, len(*filterMap)) all := query.All || query.Limit > 0 - if len(query.Filters) > 0 { - for k, v := range query.Filters { + if len((*filterMap)) > 0 { + for k, v := range *filterMap { generatedFunc, err := filters.GenerateContainerFilterFuncs(k, v, runtime) if err != nil { utils.InternalServerError(w, err) @@ -120,7 +122,7 @@ func ListContainers(w http.ResponseWriter, r *http.Request) { // Docker thinks that if status is given as an input, then we should override // the all setting and always deal with all containers. - if len(query.Filters["status"]) > 0 { + if len((*filterMap)["status"]) > 0 { all = true } if !all { diff --git a/pkg/api/handlers/compat/containers_prune.go b/pkg/api/handlers/compat/containers_prune.go index dc4d53af6..e37929d27 100644 --- a/pkg/api/handlers/compat/containers_prune.go +++ b/pkg/api/handlers/compat/containers_prune.go @@ -9,23 +9,20 @@ import ( "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/entities/reports" "github.com/containers/podman/v3/pkg/domain/filters" - "github.com/gorilla/schema" + "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" ) func PruneContainers(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - decoder := r.Context().Value("decoder").(*schema.Decoder) - - query := struct { - Filters map[string][]string `schema:"filters"` - }{} - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + filtersMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - filterFuncs := make([]libpod.ContainerFilter, 0, len(query.Filters)) - for k, v := range query.Filters { + + filterFuncs := make([]libpod.ContainerFilter, 0, len(*filtersMap)) + for k, v := range *filtersMap { generatedFunc, err := filters.GenerateContainerFilterFuncs(k, v, runtime) if err != nil { utils.InternalServerError(w, err) diff --git a/pkg/api/handlers/compat/events.go b/pkg/api/handlers/compat/events.go index 9e82831d7..dd0a9e7a9 100644 --- a/pkg/api/handlers/compat/events.go +++ b/pkg/api/handlers/compat/events.go @@ -1,69 +1,19 @@ package compat import ( - "encoding/json" - "fmt" "net/http" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -// filtersFromRequests extracts the "filters" parameter from the specified -// http.Request. The parameter can either be a `map[string][]string` as done -// in new versions of Docker and libpod, or a `map[string]map[string]bool` as -// done in older versions of Docker. We have to do a bit of Yoga to support -// both - just as Docker does as well. -// -// Please refer to https://github.com/containers/podman/issues/6899 for some -// background. -func filtersFromRequest(r *http.Request) ([]string, error) { - var ( - compatFilters map[string]map[string]bool - filters map[string][]string - libpodFilters []string - raw []byte - ) - - if _, found := r.URL.Query()["filters"]; found { - raw = []byte(r.Form.Get("filters")) - } else if _, found := r.URL.Query()["Filters"]; found { - raw = []byte(r.Form.Get("Filters")) - } else { - return []string{}, nil - } - - // Backwards compat with older versions of Docker. - if err := json.Unmarshal(raw, &compatFilters); err == nil { - for filterKey, filterMap := range compatFilters { - for filterValue, toAdd := range filterMap { - if toAdd { - libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", filterKey, filterValue)) - } - } - } - return libpodFilters, nil - } - - if err := json.Unmarshal(raw, &filters); err != nil { - return nil, err - } - - for filterKey, filterSlice := range filters { - for _, filterValue := range filterSlice { - libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", filterKey, filterValue)) - } - } - - return libpodFilters, nil -} - // NOTE: this endpoint serves both the docker-compatible one and the new libpod // one. func GetEvents(w http.ResponseWriter, r *http.Request) { @@ -92,7 +42,7 @@ func GetEvents(w http.ResponseWriter, r *http.Request) { fromStart = true } - libpodFilters, err := filtersFromRequest(r) + libpodFilters, err := util.FiltersFromRequest(r) if err != nil { utils.Error(w, "failed to parse parameters", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 7751b91a7..0a63d6e1c 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -69,7 +69,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { BuildArgs string `schema:"buildargs"` CacheFrom string `schema:"cachefrom"` Compression uint64 `schema:"compression"` - ConfigureNetwork int64 `schema:"networkmode"` + ConfigureNetwork string `schema:"networkmode"` CpuPeriod uint64 `schema:"cpuperiod"` // nolint CpuQuota int64 `schema:"cpuquota"` // nolint CpuSetCpus string `schema:"cpusetcpus"` // nolint @@ -84,7 +84,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { ForceRm bool `schema:"forcerm"` From string `schema:"from"` HTTPProxy bool `schema:"httpproxy"` - Isolation int64 `schema:"isolation"` + Isolation string `schema:"isolation"` Ignore bool `schema:"ignore"` Jobs int `schema:"jobs"` // nolint Labels string `schema:"labels"` @@ -205,9 +205,15 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { isolation := buildah.IsolationDefault */ if utils.IsLibpodRequest(r) { - // isolation = buildah.Isolation(query.Isolation) + // isolation = parseLibPodIsolation(query.Isolation) registry = "" format = query.OutputFormat + } else { + if _, found := r.URL.Query()["isolation"]; found { + if query.Isolation != "" && query.Isolation != "default" { + logrus.Debugf("invalid `isolation` parameter: %q", query.Isolation) + } + } } var additionalTags []string if len(query.Tag) > 1 { @@ -329,7 +335,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { CNIConfigDir: rtc.Network.CNIPluginDirs[0], CNIPluginPath: util.DefaultCNIPluginPath, Compression: compression, - ConfigureNetwork: buildah.NetworkConfigurationPolicy(query.ConfigureNetwork), + ConfigureNetwork: parseNetworkConfigurationPolicy(query.ConfigureNetwork), ContextDirectory: contextDirectory, Devices: devices, DropCapabilities: dropCaps, @@ -459,6 +465,40 @@ loop: } } +func parseNetworkConfigurationPolicy(network string) buildah.NetworkConfigurationPolicy { + if val, err := strconv.Atoi(network); err == nil { + return buildah.NetworkConfigurationPolicy(val) + } + switch network { + case "NetworkDefault": + return buildah.NetworkDefault + case "NetworkDisabled": + return buildah.NetworkDisabled + case "NetworkEnabled": + return buildah.NetworkEnabled + default: + return buildah.NetworkDefault + } +} + +func parseLibPodIsolation(isolation string) buildah.Isolation { // nolint + if val, err := strconv.Atoi(isolation); err == nil { + return buildah.Isolation(val) + } + switch isolation { + case "IsolationDefault", "default": + return buildah.IsolationDefault + case "IsolationOCI": + return buildah.IsolationOCI + case "IsolationChroot": + return buildah.IsolationChroot + case "IsolationOCIRootless": + return buildah.IsolationOCIRootless + default: + return buildah.IsolationDefault + } +} + func extractTarFile(r *http.Request) (string, error) { // build a home for the request body anchorDir, err := ioutil.TempDir("", "libpod_builder") diff --git a/pkg/api/handlers/compat/images_prune.go b/pkg/api/handlers/compat/images_prune.go index 63daaa780..ddf559ec6 100644 --- a/pkg/api/handlers/compat/images_prune.go +++ b/pkg/api/handlers/compat/images_prune.go @@ -8,8 +8,8 @@ import ( "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/pkg/api/handlers" "github.com/containers/podman/v3/pkg/api/handlers/utils" + "github.com/containers/podman/v3/pkg/util" "github.com/docker/docker/api/types" - "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -17,27 +17,20 @@ func PruneImages(w http.ResponseWriter, r *http.Request) { var ( filters []string ) - decoder := r.Context().Value("decoder").(*schema.Decoder) runtime := r.Context().Value("runtime").(*libpod.Runtime) - query := struct { - All bool - Filters map[string][]string `schema:"filters"` - }{ - // This is where you can override the golang default value for one of fields - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - for k, v := range query.Filters { + for k, v := range *filterMap { for _, val := range v { filters = append(filters, fmt.Sprintf("%s=%s", k, val)) } } - imagePruneReports, err := runtime.ImageRuntime().PruneImages(r.Context(), query.All, filters) + imagePruneReports, err := runtime.ImageRuntime().PruneImages(r.Context(), false, filters) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/compat/networks.go b/pkg/api/handlers/compat/networks.go index dfb1d7fda..77ed548d8 100644 --- a/pkg/api/handlers/compat/networks.go +++ b/pkg/api/handlers/compat/networks.go @@ -17,6 +17,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" networkid "github.com/containers/podman/v3/pkg/network" + "github.com/containers/podman/v3/pkg/util" "github.com/docker/docker/api/types" dockerNetwork "github.com/docker/docker/api/types/network" "github.com/gorilla/schema" @@ -181,18 +182,12 @@ func findPluginByName(plugins []*libcni.NetworkConfig, pluginType string) ([]byt func ListNetworks(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - filters, err := filtersFromRequest(r) + filterMap, err := util.PrepareFilters(r) if err != nil { - utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - filterMap := map[string][]string{} - for _, filter := range filters { - split := strings.SplitN(filter, "=", 2) - if len(split) > 1 { - filterMap[split[0]] = append(filterMap[split[0]], split[1]) - } - } + config, err := runtime.GetConfig() if err != nil { utils.InternalServerError(w, err) @@ -208,7 +203,7 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) { reports := []*types.NetworkResource{} logrus.Debugf("netNames: %q", strings.Join(netNames, ", ")) for _, name := range netNames { - report, err := getNetworkResourceByNameOrID(name, runtime, filterMap) + report, err := getNetworkResourceByNameOrID(name, runtime, *filterMap) if err != nil { utils.InternalServerError(w, err) return @@ -400,10 +395,17 @@ func Disconnect(w http.ResponseWriter, r *http.Request) { // Prune removes unused networks func Prune(w http.ResponseWriter, r *http.Request) { - // TODO Filters are not implemented runtime := r.Context().Value("runtime").(*libpod.Runtime) + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "Decode()")) + return + } + ic := abi.ContainerEngine{Libpod: runtime} - pruneOptions := entities.NetworkPruneOptions{} + pruneOptions := entities.NetworkPruneOptions{ + Filters: *filterMap, + } pruneReports, err := ic.NetworkPrune(r.Context(), pruneOptions) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) diff --git a/pkg/api/handlers/compat/volumes.go b/pkg/api/handlers/compat/volumes.go index d2febc615..42ece643b 100644 --- a/pkg/api/handlers/compat/volumes.go +++ b/pkg/api/handlers/compat/volumes.go @@ -5,7 +5,6 @@ import ( "encoding/json" "net/http" "net/url" - "strings" "time" "github.com/containers/podman/v3/libpod" @@ -14,6 +13,7 @@ import ( "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/filters" "github.com/containers/podman/v3/pkg/domain/infra/abi/parse" + "github.com/containers/podman/v3/pkg/util" docker_api_types "github.com/docker/docker/api/types" docker_api_types_volume "github.com/docker/docker/api/types/volume" "github.com/gorilla/schema" @@ -22,16 +22,10 @@ import ( func ListVolumes(w http.ResponseWriter, r *http.Request) { var ( - decoder = r.Context().Value("decoder").(*schema.Decoder) runtime = r.Context().Value("runtime").(*libpod.Runtime) ) - query := struct { - Filters map[string][]string `schema:"filters"` - }{ - // override any golang type defaults - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { + filtersMap, err := util.PrepareFilters(r) + if err != nil { utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return @@ -39,14 +33,14 @@ func ListVolumes(w http.ResponseWriter, r *http.Request) { // Reject any libpod specific filters since `GenerateVolumeFilters()` will // happily parse them for us. - for filter := range query.Filters { + for filter := range *filtersMap { if filter == "opts" { utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Errorf("unsupported libpod filters passed to docker endpoint")) return } } - volumeFilters, err := filters.GenerateVolumeFilters(query.Filters) + volumeFilters, err := filters.GenerateVolumeFilters(*filtersMap) if err != nil { utils.InternalServerError(w, err) return @@ -265,20 +259,13 @@ func PruneVolumes(w http.ResponseWriter, r *http.Request) { var ( runtime = r.Context().Value("runtime").(*libpod.Runtime) ) - filtersList, err := filtersFromRequest(r) + filterMap, err := util.PrepareFilters(r) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "Decode()")) return } - filterMap := map[string][]string{} - for _, filter := range filtersList { - split := strings.SplitN(filter, "=", 2) - if len(split) > 1 { - filterMap[split[0]] = append(filterMap[split[0]], split[1]) - } - } - f := (url.Values)(filterMap) + f := (url.Values)(*filterMap) filterFuncs, err := filters.GenerateVolumeFilters(f) if err != nil { utils.Error(w, "Something when wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to parse filters for %s", f.Encode())) diff --git a/pkg/api/handlers/libpod/containers.go b/pkg/api/handlers/libpod/containers.go index 01b9ec101..77269db8b 100644 --- a/pkg/api/handlers/libpod/containers.go +++ b/pkg/api/handlers/libpod/containers.go @@ -11,6 +11,7 @@ import ( "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -59,20 +60,21 @@ func ContainerExists(w http.ResponseWriter, r *http.Request) { func ListContainers(w http.ResponseWriter, r *http.Request) { decoder := r.Context().Value("decoder").(*schema.Decoder) query := struct { - All bool `schema:"all"` - External bool `schema:"external"` - Filters map[string][]string `schema:"filters"` - Last int `schema:"last"` // alias for limit - Limit int `schema:"limit"` - Namespace bool `schema:"namespace"` - Size bool `schema:"size"` - Sync bool `schema:"sync"` + All bool `schema:"all"` + External bool `schema:"external"` + Last int `schema:"last"` // alias for limit + Limit int `schema:"limit"` + Namespace bool `schema:"namespace"` + Size bool `schema:"size"` + Sync bool `schema:"sync"` }{ // override any golang type defaults } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + filterMap, err := util.PrepareFilters(r) + + if dErr := decoder.Decode(&query, r.URL.Query()); dErr != nil || err != nil { + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } @@ -94,7 +96,7 @@ func ListContainers(w http.ResponseWriter, r *http.Request) { opts := entities.ContainerListOptions{ All: query.All, External: query.External, - Filters: query.Filters, + Filters: *filterMap, Last: limit, Namespace: query.Namespace, // Always return Pod, should not be part of the API. diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index 83fe23621..158babcdc 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -22,6 +22,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/containers/podman/v3/pkg/errorhandling" + "github.com/containers/podman/v3/pkg/util" utils2 "github.com/containers/podman/v3/utils" "github.com/gorilla/schema" "github.com/pkg/errors" @@ -125,31 +126,32 @@ func PruneImages(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) query := struct { - All bool `schema:"all"` - Filters map[string][]string `schema:"filters"` + All bool `schema:"all"` }{ // override any golang type defaults } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + filterMap, err := util.PrepareFilters(r) + + if dErr := decoder.Decode(&query, r.URL.Query()); dErr != nil || err != nil { + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } var libpodFilters = []string{} if _, found := r.URL.Query()["filters"]; found { - dangling := query.Filters["all"] + dangling := (*filterMap)["all"] if len(dangling) > 0 { - query.All, err = strconv.ParseBool(query.Filters["all"][0]) + query.All, err = strconv.ParseBool((*filterMap)["all"][0]) if err != nil { utils.InternalServerError(w, err) return } } // dangling is special and not implemented in the libpod side of things - delete(query.Filters, "dangling") - for k, v := range query.Filters { + delete(*filterMap, "dangling") + for k, v := range *filterMap { libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", k, v[0])) } } @@ -319,18 +321,6 @@ func ExportImages(w http.ResponseWriter, r *http.Request) { func ImagesLoad(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - decoder := r.Context().Value("decoder").(*schema.Decoder) - query := struct { - Reference string `schema:"reference"` - }{ - // Add defaults here once needed. - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, - errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) - return - } tmpfile, err := ioutil.TempFile("", "libpod-images-load.tar") if err != nil { @@ -338,14 +328,15 @@ func ImagesLoad(w http.ResponseWriter, r *http.Request) { return } defer os.Remove(tmpfile.Name()) - defer tmpfile.Close() - if _, err := io.Copy(tmpfile, r.Body); err != nil && err != io.EOF { + _, err = io.Copy(tmpfile, r.Body) + tmpfile.Close() + + if err != nil && err != io.EOF { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "unable to write archive to temporary file")) return } - tmpfile.Close() loadedImage, err := runtime.LoadImage(context.Background(), tmpfile.Name(), os.Stderr, "") if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "unable to load image")) diff --git a/pkg/api/handlers/libpod/networks.go b/pkg/api/handlers/libpod/networks.go index 48cd37994..5417f778e 100644 --- a/pkg/api/handlers/libpod/networks.go +++ b/pkg/api/handlers/libpod/networks.go @@ -10,6 +10,7 @@ import ( "github.com/containers/podman/v3/pkg/api/handlers/utils" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -45,20 +46,15 @@ func CreateNetwork(w http.ResponseWriter, r *http.Request) { } func ListNetworks(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - decoder := r.Context().Value("decoder").(*schema.Decoder) - query := struct { - Filters map[string][]string `schema:"filters"` - }{ - // override any golang type defaults - } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } options := entities.NetworkListOptions{ - Filters: query.Filters, + Filters: *filterMap, } ic := abi.ContainerEngine{Libpod: runtime} reports, err := ic.NetworkList(r.Context(), options) @@ -78,7 +74,7 @@ func RemoveNetwork(w http.ResponseWriter, r *http.Request) { // override any golang type defaults } if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } @@ -111,7 +107,7 @@ func InspectNetwork(w http.ResponseWriter, r *http.Request) { // override any golang type defaults } if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } @@ -177,10 +173,18 @@ func ExistsNetwork(w http.ResponseWriter, r *http.Request) { // Prune removes unused networks func Prune(w http.ResponseWriter, r *http.Request) { - // TODO Filters are not implemented runtime := r.Context().Value("runtime").(*libpod.Runtime) + + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) + return + } + + pruneOptions := entities.NetworkPruneOptions{ + Filters: *filterMap, + } ic := abi.ContainerEngine{Libpod: runtime} - pruneOptions := entities.NetworkPruneOptions{} pruneReports, err := ic.NetworkPrune(r.Context(), pruneOptions) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) diff --git a/pkg/api/handlers/libpod/volumes.go b/pkg/api/handlers/libpod/volumes.go index a602e6744..442b53d1e 100644 --- a/pkg/api/handlers/libpod/volumes.go +++ b/pkg/api/handlers/libpod/volumes.go @@ -13,6 +13,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/filters" "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/containers/podman/v3/pkg/domain/infra/abi/parse" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -29,7 +30,7 @@ func CreateVolume(w http.ResponseWriter, r *http.Request) { } input := entities.VolumeCreateOptions{} if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } @@ -95,22 +96,16 @@ func InspectVolume(w http.ResponseWriter, r *http.Request) { func ListVolumes(w http.ResponseWriter, r *http.Request) { var ( - decoder = r.Context().Value("decoder").(*schema.Decoder) runtime = r.Context().Value("runtime").(*libpod.Runtime) ) - query := struct { - Filters map[string][]string `schema:"filters"` - }{ - // override any golang type defaults - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + filterMap, err := util.PrepareFilters(r) + if err != nil { + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - volumeFilters, err := filters.GenerateVolumeFilters(query.Filters) + volumeFilters, err := filters.GenerateVolumeFilters(*filterMap) if err != nil { utils.InternalServerError(w, err) return @@ -148,19 +143,13 @@ func PruneVolumes(w http.ResponseWriter, r *http.Request) { func pruneVolumesHelper(r *http.Request) ([]*reports.PruneReport, error) { var ( runtime = r.Context().Value("runtime").(*libpod.Runtime) - decoder = r.Context().Value("decoder").(*schema.Decoder) ) - query := struct { - Filters map[string][]string `schema:"filters"` - }{ - // override any golang type defaults - } - - if err := decoder.Decode(&query, r.URL.Query()); err != nil { + filterMap, err := util.PrepareFilters(r) + if err != nil { return nil, err } - f := (url.Values)(query.Filters) + f := (url.Values)(*filterMap) filterFuncs, err := filters.GenerateVolumeFilters(f) if err != nil { return nil, err @@ -172,6 +161,7 @@ func pruneVolumesHelper(r *http.Request) ([]*reports.PruneReport, error) { } return reports, nil } + func RemoveVolume(w http.ResponseWriter, r *http.Request) { var ( runtime = r.Context().Value("runtime").(*libpod.Runtime) @@ -184,7 +174,7 @@ func RemoveVolume(w http.ResponseWriter, r *http.Request) { } if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + utils.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } diff --git a/pkg/api/handlers/utils/images.go b/pkg/api/handlers/utils/images.go index 743629db8..da3c9e985 100644 --- a/pkg/api/handlers/utils/images.go +++ b/pkg/api/handlers/utils/images.go @@ -11,6 +11,7 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/image" + "github.com/containers/podman/v3/pkg/util" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -58,13 +59,17 @@ func GetImages(w http.ResponseWriter, r *http.Request) ([]*image.Image, error) { runtime := r.Context().Value("runtime").(*libpod.Runtime) query := struct { All bool - Filters map[string][]string `schema:"filters"` Digests bool Filter string // Docker 1.24 compatibility }{ // This is where you can override the golang default value for one of fields } + filterMap, err := util.PrepareFilters(r) + if err != nil { + return nil, err + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { return nil, err } @@ -72,12 +77,9 @@ func GetImages(w http.ResponseWriter, r *http.Request) ([]*image.Image, error) { if _, found := r.URL.Query()["digests"]; found && query.Digests { UnSupportedParameter("digests") } - var ( - images []*image.Image - err error - ) + var images []*image.Image - queryFilters := query.Filters + queryFilters := *filterMap if !IsLibpodRequest(r) && len(query.Filter) > 0 { // Docker 1.24 compatibility if queryFilters == nil { queryFilters = make(map[string][]string) diff --git a/pkg/api/server/register_images.go b/pkg/api/server/register_images.go index 3d86e5d38..423766bd8 100644 --- a/pkg/api/server/register_images.go +++ b/pkg/api/server/register_images.go @@ -810,11 +810,14 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // summary: Load image // description: Load an image (oci-archive or docker-archive) stream. // parameters: - // - in: formData + // - in: body // name: upload - // description: tarball of container image - // type: file // required: true + // description: tarball of container image + // schema: + // type: string + // consumes: + // - application/x-tar // produces: // - application/json // responses: diff --git a/pkg/api/server/register_networks.go b/pkg/api/server/register_networks.go index c54de952f..a07c6a55e 100644 --- a/pkg/api/server/register_networks.go +++ b/pkg/api/server/register_networks.go @@ -172,7 +172,6 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // name: filters // type: string // description: | - // NOT IMPLEMENTED // Filters to process on the prune list, encoded as JSON (a map[string][]string). // Available filters: // - until=<timestamp> Prune networks created before this timestamp. The <timestamp> can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. 10m, 1h30m) computed relative to the daemon machine’s time. diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go index f48b99a95..ecae22a1b 100644 --- a/pkg/bindings/containers/attach.go +++ b/pkg/bindings/containers/attach.go @@ -336,7 +336,7 @@ func attachHandleResize(ctx, winCtx context.Context, winChange chan os.Signal, i case <-winCtx.Done(): return case <-winChange: - h, w, err := terminal.GetSize(int(file.Fd())) + w, h, err := terminal.GetSize(int(file.Fd())) if err != nil { logrus.Warnf("failed to obtain TTY size: %v", err) } diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 1cbd28c37..9d77883f9 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -20,6 +20,7 @@ import ( "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/storage/pkg/fileutils" + "github.com/containers/storage/pkg/ioutils" "github.com/docker/go-units" "github.com/hashicorp/go-multierror" jsoniter "github.com/json-iterator/go" @@ -252,7 +253,11 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO logrus.Errorf("cannot tar container entries %v error: %v", entries, err) return nil, err } - defer tarfile.Close() + defer func() { + if err := tarfile.Close(); err != nil { + logrus.Errorf("%v\n", err) + } + }() containerFile, err := filepath.Abs(entries[0]) if err != nil { @@ -340,7 +345,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { gw := gzip.NewWriter(pw) tw := tar.NewWriter(gw) - var merr error + var merr *multierror.Error go func() { defer pw.Close() defer gw.Close() @@ -421,7 +426,14 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { merr = multierror.Append(merr, err) } }() - return pr, merr + rc := ioutils.NewReadCloserWrapper(pr, func() error { + if merr != nil { + merr = multierror.Append(merr, pr.Close()) + return merr.ErrorOrNil() + } + return pr.Close() + }) + return rc, nil } func parseDockerignore(root string) ([]string, error) { diff --git a/pkg/domain/entities/network.go b/pkg/domain/entities/network.go index f66a7f575..a89501664 100644 --- a/pkg/domain/entities/network.go +++ b/pkg/domain/entities/network.go @@ -92,4 +92,6 @@ type NetworkPruneReport struct { // NetworkPruneOptions describes options for pruning // unused cni networks -type NetworkPruneOptions struct{} +type NetworkPruneOptions struct { + Filters map[string][]string +} diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index 98b8f7e88..84cf03764 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -8,7 +8,6 @@ import ( "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/network" - "github.com/containers/podman/v3/pkg/timetype" "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" ) @@ -24,27 +23,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo case "label": // we have to match that all given labels exits on that container return func(c *libpod.Container) bool { - labels := c.Labels() - for _, filterValue := range filterValues { - matched := false - filterArray := strings.SplitN(filterValue, "=", 2) - filterKey := filterArray[0] - if len(filterArray) > 1 { - filterValue = filterArray[1] - } else { - filterValue = "" - } - for labelKey, labelValue := range labels { - if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { - matched = true - break - } - } - if !matched { - return false - } - } - return true + return util.MatchLabelFilters(filterValues, c.Labels()) }, nil case "name": // we only have to match one name @@ -186,18 +165,10 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo return false }, nil case "until": - if len(filterValues) != 1 { - return nil, errors.Errorf("specify exactly one timestamp for %s", filter) - } - ts, err := timetype.GetTimestamp(filterValues[0], time.Now()) - if err != nil { - return nil, err - } - seconds, nanoseconds, err := timetype.ParseTimestamps(ts, 0) + until, err := util.ComputeUntilTimestamp(filterValues) if err != nil { return nil, err } - until := time.Unix(seconds, nanoseconds) return func(c *libpod.Container) bool { if !until.IsZero() && c.CreatedTime().After((until)) { return true diff --git a/pkg/domain/filters/pods.go b/pkg/domain/filters/pods.go index 0490a4848..9a1c7d19d 100644 --- a/pkg/domain/filters/pods.go +++ b/pkg/domain/filters/pods.go @@ -114,26 +114,7 @@ func GeneratePodFilterFunc(filter string, filterValues []string) ( case "label": return func(p *libpod.Pod) bool { labels := p.Labels() - for _, filterValue := range filterValues { - matched := false - filterArray := strings.SplitN(filterValue, "=", 2) - filterKey := filterArray[0] - if len(filterArray) > 1 { - filterValue = filterArray[1] - } else { - filterValue = "" - } - for labelKey, labelValue := range labels { - if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { - matched = true - break - } - } - if !matched { - return false - } - } - return true + return util.MatchLabelFilters(filterValues, labels) }, nil case "network": return func(p *libpod.Pod) bool { diff --git a/pkg/domain/filters/volumes.go b/pkg/domain/filters/volumes.go index bc1756cf5..9b2fc5280 100644 --- a/pkg/domain/filters/volumes.go +++ b/pkg/domain/filters/volumes.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/containers/podman/v3/libpod" + "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" ) @@ -29,21 +30,9 @@ func GenerateVolumeFilters(filters url.Values) ([]libpod.VolumeFilter, error) { return v.Scope() == scopeVal }) case "label": - filterArray := strings.SplitN(val, "=", 2) - filterKey := filterArray[0] - var filterVal string - if len(filterArray) > 1 { - filterVal = filterArray[1] - } else { - filterVal = "" - } + filter := val vf = append(vf, func(v *libpod.Volume) bool { - for labelKey, labelValue := range v.Labels() { - if labelKey == filterKey && (filterVal == "" || labelValue == filterVal) { - return true - } - } - return false + return util.MatchLabelFilters([]string{filter}, v.Labels()) }) case "opt": filterArray := strings.SplitN(val, "=", 2) diff --git a/pkg/domain/infra/abi/generate.go b/pkg/domain/infra/abi/generate.go index 161becbfa..94f649e15 100644 --- a/pkg/domain/infra/abi/generate.go +++ b/pkg/domain/infra/abi/generate.go @@ -44,11 +44,10 @@ func (ic *ContainerEngine) GenerateSystemd(ctx context.Context, nameOrID string, func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, options entities.GenerateKubeOptions) (*entities.GenerateKubeReport, error) { var ( pods []*libpod.Pod - podYAML *k8sAPI.Pod - err error ctrs []*libpod.Container - servicePorts []k8sAPI.ServicePort - serviceYAML k8sAPI.Service + kubePods []*k8sAPI.Pod + kubeServices []k8sAPI.Service + content []byte ) for _, nameOrID := range nameOrIDs { // Get the container in question @@ -59,9 +58,6 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return nil, err } pods = append(pods, pod) - if len(pods) > 1 { - return nil, errors.New("can only generate single pod at a time") - } } else { if len(ctr.Dependencies()) > 0 { return nil, errors.Wrapf(define.ErrNotImplemented, "containers with dependencies") @@ -79,20 +75,29 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return nil, errors.New("cannot generate pods and containers at the same time") } - if len(pods) == 1 { - podYAML, servicePorts, err = pods[0].GenerateForKube() + if len(pods) >= 1 { + pos, svcs, err := getKubePods(pods, options.Service) + if err != nil { + return nil, err + } + + kubePods = append(kubePods, pos...) + if options.Service { + kubeServices = append(kubeServices, svcs...) + } } else { - podYAML, err = libpod.GenerateForKube(ctrs) - } - if err != nil { - return nil, err - } + po, err := libpod.GenerateForKube(ctrs) + if err != nil { + return nil, err + } - if options.Service { - serviceYAML = libpod.GenerateKubeServiceFromV1Pod(podYAML, servicePorts) + kubePods = append(kubePods, po) + if options.Service { + kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, []k8sAPI.ServicePort{})) + } } - content, err := generateKubeOutput(podYAML, &serviceYAML, options.Service) + content, err := generateKubeOutput(kubePods, kubeServices, options.Service) if err != nil { return nil, err } @@ -100,24 +105,56 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, return &entities.GenerateKubeReport{Reader: bytes.NewReader(content)}, nil } -func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasService bool) ([]byte, error) { - var ( - output []byte - marshalledPod []byte - marshalledService []byte - err error - ) +func getKubePods(pods []*libpod.Pod, getService bool) ([]*k8sAPI.Pod, []k8sAPI.Service, error) { + kubePods := make([]*k8sAPI.Pod, 0) + kubeServices := make([]k8sAPI.Service, 0) - marshalledPod, err = yaml.Marshal(podYAML) - if err != nil { - return nil, err + for _, p := range pods { + po, svc, err := p.GenerateForKube() + if err != nil { + return nil, nil, err + } + + kubePods = append(kubePods, po) + if getService { + kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, svc)) + } } - if hasService { - marshalledService, err = yaml.Marshal(serviceYAML) + return kubePods, kubeServices, nil +} + +func generateKubeOutput(kubePods []*k8sAPI.Pod, kubeServices []k8sAPI.Service, hasService bool) ([]byte, error) { + output := make([]byte, 0) + marshalledPods := make([]byte, 0) + marshalledServices := make([]byte, 0) + + for i, p := range kubePods { + if i != 0 { + marshalledPods = append(marshalledPods, []byte("---\n")...) + } + + b, err := yaml.Marshal(p) if err != nil { return nil, err } + + marshalledPods = append(marshalledPods, b...) + } + + if hasService { + for i, s := range kubeServices { + if i != 0 { + marshalledServices = append(marshalledServices, []byte("---\n")...) + } + + b, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + + marshalledServices = append(marshalledServices, b...) + } } header := `# Generation of Kubernetes YAML is still under development! @@ -133,11 +170,12 @@ func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasSer } output = append(output, []byte(fmt.Sprintf(header, podmanVersion.Version))...) - output = append(output, marshalledPod...) + // kube generate order is based on helm install order (service, pod...) if hasService { + output = append(output, marshalledServices...) output = append(output, []byte("---\n")...) - output = append(output, marshalledService...) } + output = append(output, marshalledPods...) return output, nil } diff --git a/pkg/domain/infra/abi/network.go b/pkg/domain/infra/abi/network.go index edde8ece6..1a833332c 100644 --- a/pkg/domain/infra/abi/network.go +++ b/pkg/domain/infra/abi/network.go @@ -174,17 +174,37 @@ func (ic *ContainerEngine) NetworkPrune(ctx context.Context, options entities.Ne if err != nil { return nil, err } + networks, err := network.LoadCNIConfsFromDir(network.GetCNIConfDir(runtimeConfig)) + if err != nil { + return nil, err + } + // Gather up all the non-default networks that the // containers want - usedNetworks := make(map[string]bool) + networksToKeep := make(map[string]bool) for _, c := range cons { nets, _, err := c.Networks() if err != nil { return nil, err } for _, n := range nets { - usedNetworks[n] = true + networksToKeep[n] = true + } + } + if len(options.Filters) != 0 { + for _, n := range networks { + // This network will be kept anyway + if _, found := networksToKeep[n.Name]; found { + continue + } + ok, err := network.IfPassesPruneFilter(runtimeConfig, n, options.Filters) + if err != nil { + return nil, err + } + if !ok { + networksToKeep[n.Name] = true + } } } - return network.PruneNetworks(runtimeConfig, usedNetworks) + return network.PruneNetworks(runtimeConfig, networksToKeep) } diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index efc7c86e3..7d87fc83a 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -1,6 +1,7 @@ package abi import ( + "bytes" "context" "fmt" "io" @@ -20,46 +21,79 @@ import ( "github.com/ghodss/yaml" "github.com/pkg/errors" "github.com/sirupsen/logrus" + yamlv3 "gopkg.in/yaml.v3" v1apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) func (ic *ContainerEngine) PlayKube(ctx context.Context, path string, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { - var ( - kubeObject v1.ObjectReference - ) + report := &entities.PlayKubeReport{} + validKinds := 0 + // read yaml document content, err := ioutil.ReadFile(path) if err != nil { return nil, err } - if err := yaml.Unmarshal(content, &kubeObject); err != nil { - return nil, errors.Wrapf(err, "unable to read %q as YAML", path) + // split yaml document + documentList, err := splitMultiDocYAML(content) + if err != nil { + return nil, err } - // NOTE: pkg/bindings/play is also parsing the file. - // A pkg/kube would be nice to refactor and abstract - // parts of the K8s-related code. - switch kubeObject.Kind { - case "Pod": - var podYAML v1.Pod - var podTemplateSpec v1.PodTemplateSpec - if err := yaml.Unmarshal(content, &podYAML); err != nil { - return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path) + // create pod on each document if it is a pod or deployment + // any other kube kind will be skipped + for _, document := range documentList { + kind, err := getKubeKind(document) + if err != nil { + return nil, errors.Wrapf(err, "unable to read %q as kube YAML", path) } - podTemplateSpec.ObjectMeta = podYAML.ObjectMeta - podTemplateSpec.Spec = podYAML.Spec - return ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options) - case "Deployment": - var deploymentYAML v1apps.Deployment - if err := yaml.Unmarshal(content, &deploymentYAML); err != nil { - return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path) + + switch kind { + case "Pod": + var podYAML v1.Pod + var podTemplateSpec v1.PodTemplateSpec + + if err := yaml.Unmarshal(document, &podYAML); err != nil { + return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path) + } + + podTemplateSpec.ObjectMeta = podYAML.ObjectMeta + podTemplateSpec.Spec = podYAML.Spec + + r, err := ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options) + if err != nil { + return nil, err + } + + report.Pods = append(report.Pods, r.Pods...) + validKinds++ + case "Deployment": + var deploymentYAML v1apps.Deployment + + if err := yaml.Unmarshal(document, &deploymentYAML); err != nil { + return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path) + } + + r, err := ic.playKubeDeployment(ctx, &deploymentYAML, options) + if err != nil { + return nil, err + } + + report.Pods = append(report.Pods, r.Pods...) + validKinds++ + default: + logrus.Infof("kube kind %s not supported", kind) + continue } - return ic.playKubeDeployment(ctx, &deploymentYAML, options) - default: - return nil, errors.Errorf("invalid YAML kind: %q. [Pod|Deployment] are the only supported Kubernetes Kinds", kubeObject.Kind) } + + if validKinds == 0 { + return nil, fmt.Errorf("YAML document does not contain any supported kube kind") + } + + return report, nil } func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAML *v1apps.Deployment, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { @@ -290,3 +324,45 @@ func readConfigMapFromFile(r io.Reader) (v1.ConfigMap, error) { return cm, nil } + +// splitMultiDocYAML reads mutiple documents in a YAML file and +// returns them as a list. +func splitMultiDocYAML(yamlContent []byte) ([][]byte, error) { + var documentList [][]byte + + d := yamlv3.NewDecoder(bytes.NewReader(yamlContent)) + for { + var o interface{} + // read individual document + err := d.Decode(&o) + if err == io.EOF { + break + } + if err != nil { + return nil, errors.Wrapf(err, "multi doc yaml could not be split") + } + + if o != nil { + // back to bytes + document, err := yamlv3.Marshal(o) + if err != nil { + return nil, errors.Wrapf(err, "individual doc yaml could not be marshalled") + } + + documentList = append(documentList, document) + } + } + + return documentList, nil +} + +// getKubeKind unmarshals a kube YAML document and returns its kind. +func getKubeKind(obj []byte) (string, error) { + var kubeObject v1.ObjectReference + + if err := yaml.Unmarshal(obj, &kubeObject); err != nil { + return "", err + } + + return kubeObject.Kind, nil +} diff --git a/pkg/domain/infra/abi/play_test.go b/pkg/domain/infra/abi/play_test.go index 4354a3835..bbc7c3493 100644 --- a/pkg/domain/infra/abi/play_test.go +++ b/pkg/domain/infra/abi/play_test.go @@ -89,3 +89,100 @@ data: }) } } + +func TestGetKubeKind(t *testing.T) { + tests := []struct { + name string + kubeYAML string + expectError bool + expectedErrorMsg string + expected string + }{ + { + "ValidKubeYAML", + ` +apiVersion: v1 +kind: Pod +`, + false, + "", + "Pod", + }, + { + "InvalidKubeYAML", + "InvalidKubeYAML", + true, + "cannot unmarshal", + "", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + kind, err := getKubeKind([]byte(test.kubeYAML)) + if test.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.expectedErrorMsg) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, kind) + } + }) + } +} + +func TestSplitMultiDocYAML(t *testing.T) { + tests := []struct { + name string + kubeYAML string + expectError bool + expectedErrorMsg string + expected int + }{ + { + "ValidNumberOfDocs", + ` +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +`, + false, + "", + 3, + }, + { + "InvalidMultiDocYAML", + ` +apiVersion: v1 +kind: Pod +--- +apiVersion: v1 +kind: Pod +- +`, + true, + "multi doc yaml could not be split", + 0, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + docs, err := splitMultiDocYAML([]byte(test.kubeYAML)) + if test.expectError { + assert.Error(t, err) + assert.Contains(t, err.Error(), test.expectedErrorMsg) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, len(docs)) + } + }) + } +} diff --git a/pkg/machine/config.go b/pkg/machine/config.go new file mode 100644 index 000000000..4933deee8 --- /dev/null +++ b/pkg/machine/config.go @@ -0,0 +1,120 @@ +package machine + +import ( + "net" + "net/url" + "os" + "path/filepath" + + "github.com/containers/storage/pkg/homedir" +) + +type InitOptions struct { + Name string + CPUS uint64 + Memory uint64 + IgnitionPath string + ImagePath string + Username string + URI url.URL + IsDefault bool + //KernelPath string + //Devices []VMDevices +} + +type RemoteConnectionType string + +var ( + SSHRemoteConnection RemoteConnectionType = "ssh" + DefaultIgnitionUserName = "core" +) + +type Download struct { + Arch string + Artifact string + CompressionType string + Format string + ImageName string `json:"image_name"` + LocalPath string + LocalUncompressedFile string + Sha256sum string + URL *url.URL + VMName string +} + +type SSHOptions struct { + Execute bool + Args []string +} +type StartOptions struct{} + +type StopOptions struct{} + +type RemoveOptions struct { + Force bool + SaveKeys bool + SaveImage bool + SaveIgnition bool +} + +type VM interface { + Init(opts InitOptions) error + Remove(name string, opts RemoveOptions) (string, func() error, error) + SSH(name string, opts SSHOptions) error + Start(name string, opts StartOptions) error + Stop(name string, opts StopOptions) error +} + +type DistributionDownload interface { + DownloadImage() error + Get() *Download +} + +func (rc RemoteConnectionType) MakeSSHURL(host, path, port, userName string) url.URL { + userInfo := url.User(userName) + uri := url.URL{ + Scheme: "ssh", + Opaque: "", + User: userInfo, + Host: host, + Path: path, + RawPath: "", + ForceQuery: false, + RawQuery: "", + Fragment: "", + } + if len(port) > 0 { + uri.Host = net.JoinHostPort(uri.Hostname(), port) + } + return uri +} + +// GetDataDir returns the filepath where vm images should +// live for podman-machine +func GetDataDir(vmType string) (string, error) { + data, err := homedir.GetDataHome() + if err != nil { + return "", err + } + dataDir := filepath.Join(data, "containers", "podman", "machine", vmType) + if _, err := os.Stat(dataDir); !os.IsNotExist(err) { + return dataDir, nil + } + mkdirErr := os.MkdirAll(dataDir, 0755) + return dataDir, mkdirErr +} + +// GetConfigDir returns the filepath to where configuration +// files for podman-machine should live +func GetConfDir(vmType string) (string, error) { + conf, err := homedir.GetConfigHome() + if err != nil { + return "", err + } + confDir := filepath.Join(conf, "containers", "podman", "machine", vmType) + if _, err := os.Stat(confDir); !os.IsNotExist(err) { + return confDir, nil + } + mkdirErr := os.MkdirAll(confDir, 0755) + return confDir, mkdirErr +} diff --git a/pkg/machine/connection.go b/pkg/machine/connection.go new file mode 100644 index 000000000..e3985d8ac --- /dev/null +++ b/pkg/machine/connection.go @@ -0,0 +1,50 @@ +package machine + +import ( + "fmt" + + "github.com/containers/common/pkg/config" + "github.com/pkg/errors" +) + +func AddConnection(uri fmt.Stringer, name, identity string, isDefault bool) error { + if len(identity) < 1 { + return errors.New("identity must be defined") + } + cfg, err := config.ReadCustomConfig() + if err != nil { + return err + } + if _, ok := cfg.Engine.ServiceDestinations[name]; ok { + return errors.New("cannot overwrite connection") + } + if isDefault { + cfg.Engine.ActiveService = name + } + dst := config.Destination{ + URI: uri.String(), + } + dst.Identity = identity + if cfg.Engine.ServiceDestinations == nil { + cfg.Engine.ServiceDestinations = map[string]config.Destination{ + name: dst, + } + cfg.Engine.ActiveService = name + } else { + cfg.Engine.ServiceDestinations[name] = dst + } + return cfg.Write() +} + +func RemoveConnection(name string) error { + cfg, err := config.ReadCustomConfig() + if err != nil { + return err + } + if _, ok := cfg.Engine.ServiceDestinations[name]; ok { + delete(cfg.Engine.ServiceDestinations, name) + } else { + return errors.Errorf("unable to find connection named %q", name) + } + return cfg.Write() +} diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go new file mode 100644 index 000000000..0c6a2485e --- /dev/null +++ b/pkg/machine/fcos.go @@ -0,0 +1,160 @@ +package machine + +import ( + "crypto/sha256" + "io" + "io/ioutil" + url2 "net/url" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/containers/storage/pkg/archive" + digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" +) + +// These should eventually be moved into machine/qemu as +// they are specific to running qemu +var ( + artifact string = "qemu" + Format string = "qcow2.xz" +) + +type FcosDownload struct { + Download +} + +func NewFcosDownloader(vmType, vmName string) (DistributionDownload, error) { + info, err := getFCOSDownload() + if err != nil { + return nil, err + } + urlSplit := strings.Split(info.Location, "/") + imageName := urlSplit[len(urlSplit)-1] + url, err := url2.Parse(info.Location) + if err != nil { + return nil, err + } + + dataDir, err := GetDataDir(vmType) + if err != nil { + return nil, err + } + + fcd := FcosDownload{ + Download: Download{ + Arch: getFcosArch(), + Artifact: artifact, + Format: Format, + ImageName: imageName, + LocalPath: filepath.Join(dataDir, imageName), + Sha256sum: info.Sha256Sum, + URL: url, + VMName: vmName, + }, + } + fcd.Download.LocalUncompressedFile = fcd.getLocalUncompressedName() + return fcd, nil +} + +func (f FcosDownload) getLocalUncompressedName() string { + uncompressedFilename := filepath.Join(filepath.Dir(f.LocalPath), f.VMName+"_"+f.ImageName) + return strings.TrimSuffix(uncompressedFilename, ".xz") +} + +func (f FcosDownload) DownloadImage() error { + // check if the latest image is already present + ok, err := UpdateAvailable(&f.Download) + if err != nil { + return err + } + if !ok { + if err := DownloadVMImage(f.URL, f.LocalPath); err != nil { + return err + } + } + uncompressedFileWriter, err := os.OpenFile(f.getLocalUncompressedName(), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return err + } + sourceFile, err := ioutil.ReadFile(f.LocalPath) + if err != nil { + return err + } + compressionType := archive.DetectCompression(sourceFile) + f.CompressionType = compressionType.Extension() + + switch f.CompressionType { + case "tar.xz": + return decompressXZ(f.LocalPath, uncompressedFileWriter) + default: + // File seems to be uncompressed, make a copy + if err := copyFile(f.LocalPath, uncompressedFileWriter); err != nil { + return err + } + } + return nil +} + +func copyFile(src string, dest *os.File) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer func() { + if err := source.Close(); err != nil { + logrus.Error(err) + } + }() + _, err = io.Copy(dest, source) + return err +} + +func (f FcosDownload) Get() *Download { + return &f.Download +} + +type fcosDownloadInfo struct { + CompressionType string + Location string + Release string + Sha256Sum string +} + +func UpdateAvailable(d *Download) (bool, error) { + // check the sha of the local image if it exists + // get the sha of the remote image + // == dont bother to pull + files, err := ioutil.ReadDir(filepath.Dir(d.LocalPath)) + if err != nil { + return false, err + } + for _, file := range files { + if filepath.Base(d.LocalPath) == file.Name() { + b, err := ioutil.ReadFile(d.LocalPath) + if err != nil { + return false, err + } + s := sha256.Sum256(b) + sum := digest.NewDigestFromBytes(digest.SHA256, s[:]) + if sum.Encoded() == d.Sha256sum { + return true, nil + } + } + } + return false, nil +} + +func getFcosArch() string { + var arch string + // TODO fill in more architectures + switch runtime.GOARCH { + case "arm64": + arch = "aarch64" + default: + arch = "x86_64" + } + return arch +} diff --git a/pkg/machine/fcos_amd64.go b/pkg/machine/fcos_amd64.go new file mode 100644 index 000000000..36676405a --- /dev/null +++ b/pkg/machine/fcos_amd64.go @@ -0,0 +1,68 @@ +package machine + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/coreos/stream-metadata-go/fedoracoreos" + "github.com/coreos/stream-metadata-go/stream" + "github.com/sirupsen/logrus" +) + +// This should get Exported and stay put as it will apply to all fcos downloads +// getFCOS parses fedoraCoreOS's stream and returns the image download URL and the release version +func getFCOSDownload() (*fcosDownloadInfo, error) { + var ( + fcosstable stream.Stream + ) + streamurl := fedoracoreos.GetStreamURL(fedoracoreos.StreamNext) + resp, err := http.Get(streamurl.String()) + if err != nil { + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + + if err := json.Unmarshal(body, &fcosstable); err != nil { + return nil, err + } + arch, ok := fcosstable.Architectures[getFcosArch()] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no targetArch in stream") + } + artifacts := arch.Artifacts + if artifacts == nil { + return nil, fmt.Errorf("unable to pull VM image: no artifact in stream") + } + qemu, ok := artifacts[artifact] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qemu artifact in stream") + } + formats := qemu.Formats + if formats == nil { + return nil, fmt.Errorf("unable to pull VM image: no formats in stream") + } + qcow, ok := formats[Format] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qcow2.xz format in stream") + } + disk := qcow.Disk + if disk == nil { + return nil, fmt.Errorf("unable to pull VM image: no disk in stream") + } + return &fcosDownloadInfo{ + Location: disk.Location, + Release: qemu.Release, + Sha256Sum: disk.Sha256, + CompressionType: "xz", + }, nil +} diff --git a/pkg/machine/fcos_arm64.go b/pkg/machine/fcos_arm64.go new file mode 100644 index 000000000..ab50ca874 --- /dev/null +++ b/pkg/machine/fcos_arm64.go @@ -0,0 +1,169 @@ +package machine + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/sirupsen/logrus" +) + +const aarchBaseURL = "https://fedorapeople.org/groups/fcos-images/builds/latest/aarch64/" + +// Total hack until automation is possible. +// We need a proper json file at least to automate +func getFCOSDownload() (*fcosDownloadInfo, error) { + + meta := Build{} + fmt.Println(aarchBaseURL + "meta.json") + resp, err := http.Get(aarchBaseURL + "meta.json") + if err != nil { + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + if err := json.Unmarshal(body, &meta); err != nil { + return nil, err + } + return &fcosDownloadInfo{ + Location: "https://fedorapeople.org/groups/fcos-images/builds/latest/aarch64/fedora-coreos-33.20210310.dev.0-qemu.aarch64.qcow2", + Release: "", + Sha256Sum: meta.BuildArtifacts.Qemu.Sha256, + }, nil +} + +/* + All of this can be nuked when fcos upstream generates a proper meta data file for aarch. +*/ +type AliyunImage struct { + ImageID string `json:"id"` + Region string `json:"name"` +} + +type Amis struct { + Hvm string `json:"hvm"` + Region string `json:"name"` + Snapshot string `json:"snapshot"` +} + +type Artifact struct { + Path string `json:"path"` + Sha256 string `json:"sha256"` + SizeInBytes float64 `json:"size,omitempty"` + UncompressedSha256 string `json:"uncompressed-sha256,omitempty"` + UncompressedSize int `json:"uncompressed-size,omitempty"` +} + +type Build struct { + AlibabaAliyunUploads []AliyunImage `json:"aliyun,omitempty"` + Amis []Amis `json:"amis,omitempty"` + Architecture string `json:"coreos-assembler.basearch,omitempty"` + Azure *Cloudartifact `json:"azure,omitempty"` + BuildArtifacts *BuildArtifacts `json:"images,omitempty"` + BuildID string `json:"buildid"` + BuildRef string `json:"ref,omitempty"` + BuildSummary string `json:"summary"` + BuildTimeStamp string `json:"coreos-assembler.build-timestamp,omitempty"` + BuildURL string `json:"build-url,omitempty"` + ConfigGitRev string `json:"coreos-assembler.config-gitrev,omitempty"` + ContainerConfigGit *Git `json:"coreos-assembler.container-config-git,omitempty"` + CoreOsSource string `json:"coreos-assembler.code-source,omitempty"` + CosaContainerImageGit *Git `json:"coreos-assembler.container-image-git,omitempty"` + CosaDelayedMetaMerge bool `json:"coreos-assembler.delayed-meta-merge,omitempty"` + CosaImageChecksum string `json:"coreos-assembler.image-config-checksum,omitempty"` + CosaImageVersion int `json:"coreos-assembler.image-genver,omitempty"` + Extensions *Extensions `json:"extensions,omitempty"` + FedoraCoreOsParentCommit string `json:"fedora-coreos.parent-commit,omitempty"` + FedoraCoreOsParentVersion string `json:"fedora-coreos.parent-version,omitempty"` + Gcp *Gcp `json:"gcp,omitempty"` + GitDirty string `json:"coreos-assembler.config-dirty,omitempty"` + ImageInputChecksum string `json:"coreos-assembler.image-input-checksum,omitempty"` + InputHasOfTheRpmOstree string `json:"rpm-ostree-inputhash"` + MetaStamp float64 `json:"coreos-assembler.meta-stamp,omitempty"` + Name string `json:"name"` + Oscontainer *Image `json:"oscontainer,omitempty"` + OstreeCommit string `json:"ostree-commit"` + OstreeContentBytesWritten int `json:"ostree-content-bytes-written,omitempty"` + OstreeContentChecksum string `json:"ostree-content-checksum"` + OstreeNCacheHits int `json:"ostree-n-cache-hits,omitempty"` + OstreeNContentTotal int `json:"ostree-n-content-total,omitempty"` + OstreeNContentWritten int `json:"ostree-n-content-written,omitempty"` + OstreeNMetadataTotal int `json:"ostree-n-metadata-total,omitempty"` + OstreeNMetadataWritten int `json:"ostree-n-metadata-written,omitempty"` + OstreeTimestamp string `json:"ostree-timestamp"` + OstreeVersion string `json:"ostree-version"` + OverridesActive bool `json:"coreos-assembler.overrides-active,omitempty"` + PkgdiffAgainstParent PackageSetDifferences `json:"parent-pkgdiff,omitempty"` + PkgdiffBetweenBuilds PackageSetDifferences `json:"pkgdiff,omitempty"` + ReleasePayload *Image `json:"release-payload,omitempty"` +} + +type BuildArtifacts struct { + Aliyun *Artifact `json:"aliyun,omitempty"` + Aws *Artifact `json:"aws,omitempty"` + Azure *Artifact `json:"azure,omitempty"` + AzureStack *Artifact `json:"azurestack,omitempty"` + Dasd *Artifact `json:"dasd,omitempty"` + DigitalOcean *Artifact `json:"digitalocean,omitempty"` + Exoscale *Artifact `json:"exoscale,omitempty"` + Gcp *Artifact `json:"gcp,omitempty"` + IbmCloud *Artifact `json:"ibmcloud,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Iso *Artifact `json:"iso,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + LiveInitramfs *Artifact `json:"live-initramfs,omitempty"` + LiveIso *Artifact `json:"live-iso,omitempty"` + LiveKernel *Artifact `json:"live-kernel,omitempty"` + LiveRootfs *Artifact `json:"live-rootfs,omitempty"` + Metal *Artifact `json:"metal,omitempty"` + Metal4KNative *Artifact `json:"metal4k,omitempty"` + OpenStack *Artifact `json:"openstack,omitempty"` + Ostree Artifact `json:"ostree"` + Qemu *Artifact `json:"qemu,omitempty"` + Vmware *Artifact `json:"vmware,omitempty"` + Vultr *Artifact `json:"vultr,omitempty"` +} + +type Cloudartifact struct { + Image string `json:"image"` + URL string `json:"url"` +} + +type Extensions struct { + Manifest map[string]interface{} `json:"manifest"` + Path string `json:"path"` + RpmOstreeState string `json:"rpm-ostree-state"` + Sha256 string `json:"sha256"` +} + +type Gcp struct { + ImageFamily string `json:"family,omitempty"` + ImageName string `json:"image"` + ImageProject string `json:"project,omitempty"` + URL string `json:"url"` +} + +type Git struct { + Branch string `json:"branch,omitempty"` + Commit string `json:"commit"` + Dirty string `json:"dirty,omitempty"` + Origin string `json:"origin"` +} + +type Image struct { + Comment string `json:"comment,omitempty"` + Digest string `json:"digest"` + Image string `json:"image"` +} + +type Items interface{} + +type PackageSetDifferences []Items diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go new file mode 100644 index 000000000..ff79d5afb --- /dev/null +++ b/pkg/machine/ignition.go @@ -0,0 +1,151 @@ +package machine + +import ( + "encoding/json" + "io/ioutil" +) + +/* + If this file gets too nuts, we can perhaps use existing go code + to create ignition files. At this point, the file is so simple + that I chose to use structs and not import any code as I was + concerned (unsubstantiated) about too much bloat coming in. + + https://github.com/openshift/machine-config-operator/blob/master/pkg/server/server.go +*/ + +// Convenience function to convert int to ptr +func intToPtr(i int) *int { + return &i +} + +// Convenience function to convert string to ptr +func strToPtr(s string) *string { + return &s +} + +// Convenience function to convert bool to ptr +func boolToPtr(b bool) *bool { + return &b +} + +func getNodeUsr(usrName string) NodeUser { + return NodeUser{Name: &usrName} +} + +func getNodeGrp(grpName string) NodeGroup { + return NodeGroup{Name: &grpName} +} + +// NewIgnitionFile +func NewIgnitionFile(name, key, writePath string) error { + if len(name) < 1 { + name = DefaultIgnitionUserName + } + ignVersion := Ignition{ + Version: "3.2.0", + } + + ignPassword := Passwd{ + Users: []PasswdUser{{ + Name: name, + SSHAuthorizedKeys: []SSHAuthorizedKey{SSHAuthorizedKey(key)}, + }}, + } + + ignStorage := Storage{ + Directories: getDirs(name), + Files: getFiles(name), + Links: getLinks(name), + } + ignSystemd := Systemd{ + Units: []Unit{ + { + Enabled: boolToPtr(true), + Name: "podman.socket", + }}} + + ignConfig := Config{ + Ignition: ignVersion, + Passwd: ignPassword, + Storage: ignStorage, + Systemd: ignSystemd, + } + b, err := json.Marshal(ignConfig) + if err != nil { + return err + } + return ioutil.WriteFile(writePath, b, 0644) +} + +func getDirs(usrName string) []Directory { + // Ignition has a bug/feature? where if you make a series of dirs + // in one swoop, then the leading dirs are creates as root. + newDirs := []string{ + "/home/" + usrName + "/.config", + "/home/" + usrName + "/.config/systemd", + "/home/" + usrName + "/.config/systemd/user", + "/home/" + usrName + "/.config/systemd/user/default.target.wants", + } + var ( + dirs = make([]Directory, len(newDirs)) + ) + for i, d := range newDirs { + newDir := Directory{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: d, + User: getNodeUsr(usrName), + }, + DirectoryEmbedded1: DirectoryEmbedded1{Mode: intToPtr(493)}, + } + dirs[i] = newDir + } + return dirs +} + +func getFiles(usrName string) []File { + var ( + files []File + ) + // Add a fake systemd service to get the user socket rolling + files = append(files, File{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/home/" + usrName + "/.config/systemd/user/linger-example.service", + User: getNodeUsr(usrName), + }, + FileEmbedded1: FileEmbedded1{ + Append: nil, + Contents: Resource{ + Source: strToPtr("data:,%5BUnit%5D%0ADescription%3DA%20systemd%20user%20unit%20demo%0AAfter%3Dnetwork-online.target%0AWants%3Dnetwork-online.target%20podman.socket%0A%5BService%5D%0AExecStart%3D%2Fusr%2Fbin%2Fsleep%20infinity%0A"), + }, + Mode: intToPtr(484), + }, + }) + + // Add a file into linger + files = append(files, File{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/var/lib/systemd/linger/core", + User: getNodeUsr(usrName), + }, + FileEmbedded1: FileEmbedded1{Mode: intToPtr(420)}, + }) + return files +} + +func getLinks(usrName string) []Link { + return []Link{{ + Node: Node{ + Group: getNodeGrp(usrName), + Path: "/home/" + usrName + "/.config/systemd/user/default.target.wants/linger-example.service", + User: getNodeUsr(usrName), + }, + LinkEmbedded1: LinkEmbedded1{ + Hard: boolToPtr(false), + Target: "/home/" + usrName + "/.config/systemd/user/linger-example.service", + }, + }} +} diff --git a/pkg/machine/ignition_schema.go b/pkg/machine/ignition_schema.go new file mode 100644 index 000000000..9dbd90ba4 --- /dev/null +++ b/pkg/machine/ignition_schema.go @@ -0,0 +1,251 @@ +package machine + +/* + This file was taken from https://github.com/coreos/ignition/blob/master/config/v3_2/types/schema.go in an effort to + use more of the core-os structs but not fully commit to bringing their api in. + + // generated by "schematyper --package=types config/v3_2/schema/ignition.json -o config/v3_2/types/ignition_schema.go --root-type=Config" -- DO NOT EDIT +*/ + +type Clevis struct { + Custom *Custom `json:"custom,omitempty"` + Tang []Tang `json:"tang,omitempty"` + Threshold *int `json:"threshold,omitempty"` + Tpm2 *bool `json:"tpm2,omitempty"` +} + +type Config struct { + Ignition Ignition `json:"ignition"` + Passwd Passwd `json:"passwd,omitempty"` + Storage Storage `json:"storage,omitempty"` + Systemd Systemd `json:"systemd,omitempty"` +} + +type Custom struct { + Config string `json:"config"` + NeedsNetwork *bool `json:"needsNetwork,omitempty"` + Pin string `json:"pin"` +} + +type Device string + +type Directory struct { + Node + DirectoryEmbedded1 +} + +type DirectoryEmbedded1 struct { + Mode *int `json:"mode,omitempty"` +} + +type Disk struct { + Device string `json:"device"` + Partitions []Partition `json:"partitions,omitempty"` + WipeTable *bool `json:"wipeTable,omitempty"` +} + +type Dropin struct { + Contents *string `json:"contents,omitempty"` + Name string `json:"name"` +} + +type File struct { + Node + FileEmbedded1 +} + +type FileEmbedded1 struct { + Append []Resource `json:"append,omitempty"` + Contents Resource `json:"contents,omitempty"` + Mode *int `json:"mode,omitempty"` +} + +type Filesystem struct { + Device string `json:"device"` + Format *string `json:"format,omitempty"` + Label *string `json:"label,omitempty"` + MountOptions []MountOption `json:"mountOptions,omitempty"` + Options []FilesystemOption `json:"options,omitempty"` + Path *string `json:"path,omitempty"` + UUID *string `json:"uuid,omitempty"` + WipeFilesystem *bool `json:"wipeFilesystem,omitempty"` +} + +type FilesystemOption string + +type Group string + +type HTTPHeader struct { + Name string `json:"name"` + Value *string `json:"value,omitempty"` +} + +type HTTPHeaders []HTTPHeader + +type Ignition struct { + Config IgnitionConfig `json:"config,omitempty"` + Proxy Proxy `json:"proxy,omitempty"` + Security Security `json:"security,omitempty"` + Timeouts Timeouts `json:"timeouts,omitempty"` + Version string `json:"version,omitempty"` +} + +type IgnitionConfig struct { + Merge []Resource `json:"merge,omitempty"` + Replace Resource `json:"replace,omitempty"` +} + +type Link struct { + Node + LinkEmbedded1 +} + +type LinkEmbedded1 struct { + Hard *bool `json:"hard,omitempty"` + Target string `json:"target"` +} + +type Luks struct { + Clevis *Clevis `json:"clevis,omitempty"` + Device *string `json:"device,omitempty"` + KeyFile Resource `json:"keyFile,omitempty"` + Label *string `json:"label,omitempty"` + Name string `json:"name"` + Options []LuksOption `json:"options,omitempty"` + UUID *string `json:"uuid,omitempty"` + WipeVolume *bool `json:"wipeVolume,omitempty"` +} + +type LuksOption string + +type MountOption string + +type NoProxyItem string + +type Node struct { + Group NodeGroup `json:"group,omitempty"` + Overwrite *bool `json:"overwrite,omitempty"` + Path string `json:"path"` + User NodeUser `json:"user,omitempty"` +} + +type NodeGroup struct { + ID *int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type NodeUser struct { + ID *int `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type Partition struct { + GUID *string `json:"guid,omitempty"` + Label *string `json:"label,omitempty"` + Number int `json:"number,omitempty"` + Resize *bool `json:"resize,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + SizeMiB *int `json:"sizeMiB,omitempty"` + StartMiB *int `json:"startMiB,omitempty"` + TypeGUID *string `json:"typeGuid,omitempty"` + WipePartitionEntry *bool `json:"wipePartitionEntry,omitempty"` +} + +type Passwd struct { + Groups []PasswdGroup `json:"groups,omitempty"` + Users []PasswdUser `json:"users,omitempty"` +} + +type PasswdGroup struct { + Gid *int `json:"gid,omitempty"` + Name string `json:"name"` + PasswordHash *string `json:"passwordHash,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + System *bool `json:"system,omitempty"` +} + +type PasswdUser struct { + Gecos *string `json:"gecos,omitempty"` + Groups []Group `json:"groups,omitempty"` + HomeDir *string `json:"homeDir,omitempty"` + Name string `json:"name"` + NoCreateHome *bool `json:"noCreateHome,omitempty"` + NoLogInit *bool `json:"noLogInit,omitempty"` + NoUserGroup *bool `json:"noUserGroup,omitempty"` + PasswordHash *string `json:"passwordHash,omitempty"` + PrimaryGroup *string `json:"primaryGroup,omitempty"` + SSHAuthorizedKeys []SSHAuthorizedKey `json:"sshAuthorizedKeys,omitempty"` + Shell *string `json:"shell,omitempty"` + ShouldExist *bool `json:"shouldExist,omitempty"` + System *bool `json:"system,omitempty"` + UID *int `json:"uid,omitempty"` +} + +type Proxy struct { + HTTPProxy *string `json:"httpProxy,omitempty"` + HTTPSProxy *string `json:"httpsProxy,omitempty"` + NoProxy []NoProxyItem `json:"noProxy,omitempty"` +} + +type Raid struct { + Devices []Device `json:"devices"` + Level string `json:"level"` + Name string `json:"name"` + Options []RaidOption `json:"options,omitempty"` + Spares *int `json:"spares,omitempty"` +} + +type RaidOption string + +type Resource struct { + Compression *string `json:"compression,omitempty"` + HTTPHeaders HTTPHeaders `json:"httpHeaders,omitempty"` + Source *string `json:"source,omitempty"` + Verification Verification `json:"verification,omitempty"` +} + +type SSHAuthorizedKey string + +type Security struct { + TLS TLS `json:"tls,omitempty"` +} + +type Storage struct { + Directories []Directory `json:"directories,omitempty"` + Disks []Disk `json:"disks,omitempty"` + Files []File `json:"files,omitempty"` + Filesystems []Filesystem `json:"filesystems,omitempty"` + Links []Link `json:"links,omitempty"` + Luks []Luks `json:"luks,omitempty"` + Raid []Raid `json:"raid,omitempty"` +} + +type Systemd struct { + Units []Unit `json:"units,omitempty"` +} + +type TLS struct { + CertificateAuthorities []Resource `json:"certificateAuthorities,omitempty"` +} + +type Tang struct { + Thumbprint *string `json:"thumbprint,omitempty"` + URL string `json:"url,omitempty"` +} + +type Timeouts struct { + HTTPResponseHeaders *int `json:"httpResponseHeaders,omitempty"` + HTTPTotal *int `json:"httpTotal,omitempty"` +} + +type Unit struct { + Contents *string `json:"contents,omitempty"` + Dropins []Dropin `json:"dropins,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Mask *bool `json:"mask,omitempty"` + Name string `json:"name"` +} + +type Verification struct { + Hash *string `json:"hash,omitempty"` +} diff --git a/pkg/machine/keys.go b/pkg/machine/keys.go new file mode 100644 index 000000000..907e28f55 --- /dev/null +++ b/pkg/machine/keys.go @@ -0,0 +1,25 @@ +package machine + +import ( + "io/ioutil" + "os/exec" + "strings" +) + +// CreateSSHKeys makes a priv and pub ssh key for interacting +// the a VM. +func CreateSSHKeys(writeLocation string) (string, error) { + if err := generatekeys(writeLocation); err != nil { + return "", err + } + b, err := ioutil.ReadFile(writeLocation + ".pub") + if err != nil { + return "", err + } + return strings.TrimSuffix(string(b), "\n"), nil +} + +// generatekeys creates an ed25519 set of keys +func generatekeys(writeLocation string) error { + return exec.Command("ssh-keygen", "-N", "", "-t", "ed25519", "-f", writeLocation).Run() +} diff --git a/pkg/machine/libvirt/config.go b/pkg/machine/libvirt/config.go new file mode 100644 index 000000000..903f15fbc --- /dev/null +++ b/pkg/machine/libvirt/config.go @@ -0,0 +1,4 @@ +package libvirt + +type MachineVM struct { +} diff --git a/pkg/machine/libvirt/machine.go b/pkg/machine/libvirt/machine.go new file mode 100644 index 000000000..c38f63853 --- /dev/null +++ b/pkg/machine/libvirt/machine.go @@ -0,0 +1,15 @@ +package libvirt + +import "github.com/containers/podman/v3/pkg/machine" + +func (v *MachineVM) Init(name string, opts machine.InitOptions) error { + return nil +} + +func (v *MachineVM) Start(name string) error { + return nil +} + +func (v *MachineVM) Stop(name string) error { + return nil +} diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go new file mode 100644 index 000000000..39dde15b8 --- /dev/null +++ b/pkg/machine/pull.go @@ -0,0 +1,97 @@ +package machine + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "github.com/sirupsen/logrus" + "github.com/vbauerster/mpb/v6" + "github.com/vbauerster/mpb/v6/decor" +) + +// DownloadVMImage downloads a VM image from url to given path +// with download status +func DownloadVMImage(downloadURL fmt.Stringer, localImagePath string) error { + out, err := os.Create(localImagePath) + if err != nil { + return err + } + defer func() { + if err := out.Close(); err != nil { + logrus.Error(err) + } + }() + + resp, err := http.Get(downloadURL.String()) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + logrus.Error(err) + } + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error downloading VM image: %s", resp.Status) + } + size := resp.ContentLength + urlSplit := strings.Split(downloadURL.String(), "/") + prefix := "Downloading VM image: " + urlSplit[len(urlSplit)-1] + onComplete := prefix + ": done" + + p := mpb.New( + mpb.WithWidth(60), + mpb.WithRefreshRate(180*time.Millisecond), + ) + + bar := p.AddBar(size, + mpb.BarFillerClearOnComplete(), + mpb.PrependDecorators( + decor.OnComplete(decor.Name(prefix), onComplete), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""), + ), + ) + + proxyReader := bar.ProxyReader(resp.Body) + defer func() { + if err := proxyReader.Close(); err != nil { + logrus.Error(err) + } + }() + + if _, err := io.Copy(out, proxyReader); err != nil { + return err + } + + p.Wait() + return nil +} + +// Will error out if file without .xz already exists +// Maybe extracting then renameing is a good idea here.. +// depends on xz: not pre-installed on mac, so it becomes a brew dependecy +func decompressXZ(src string, output io.Writer) error { + fmt.Println("Extracting compressed file") + cmd := exec.Command("xzcat", "-k", src) + //cmd := exec.Command("xz", "-d", "-k", "-v", src) + stdOut, err := cmd.StdoutPipe() + if err != nil { + return err + } + //cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + go func() { + if _, err := io.Copy(output, stdOut); err != nil { + logrus.Error(err) + } + }() + return cmd.Run() +} diff --git a/pkg/machine/qemu/config.go b/pkg/machine/qemu/config.go new file mode 100644 index 000000000..e4687914d --- /dev/null +++ b/pkg/machine/qemu/config.go @@ -0,0 +1,43 @@ +package qemu + +import "time" + +type MachineVM struct { + // CPUs to be assigned to the VM + CPUs uint64 + // The command line representation of the qemu command + CmdLine []string + // IdentityPath is the fq path to the ssh priv key + IdentityPath string + // IgnitionFilePath is the fq path to the .ign file + IgnitionFilePath string + // ImagePath is the fq path to + ImagePath string + // Memory in megabytes assigned to the vm + Memory uint64 + // Name of the vm + Name string + // SSH port for user networking + Port int + // QMPMonitor is the qemu monitor object for sending commands + QMPMonitor Monitor + // RemoteUsername of the vm user + RemoteUsername string +} + +type Monitor struct { + // Address portion of the qmp monitor (/tmp/tmp.sock) + Address string + // Network portion of the qmp monitor (unix) + Network string + // Timeout in seconds for qmp monitor transactions + Timeout time.Duration +} + +var ( + // defaultQMPTimeout is the timeout duration for the + // qmp monitor interactions + defaultQMPTimeout time.Duration = 2 * time.Second + // defaultRemoteUser describes the ssh username default + defaultRemoteUser = "core" +) diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go new file mode 100644 index 000000000..b97eb991a --- /dev/null +++ b/pkg/machine/qemu/machine.go @@ -0,0 +1,325 @@ +package qemu + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" + + "github.com/containers/podman/v3/utils" + + "github.com/containers/podman/v3/pkg/machine" + "github.com/containers/storage/pkg/homedir" + "github.com/digitalocean/go-qemu/qmp" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var ( + // vmtype refers to qemu (vs libvirt, krun, etc) + vmtype = "qemu" + // qemuCommon are the common command line arguments between the arches + //qemuCommon = []string{"-cpu", "host", "-qmp", "unix://tmp/qmp.sock,server,nowait"} + //qemuCommon = []string{"-cpu", "host", "-qmp", "tcp:localhost:4444,server,nowait"} +) + +// NewMachine initializes an instance of a virtual machine based on the qemu +// virtualization. +func NewMachine(opts machine.InitOptions) (machine.VM, error) { + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return nil, err + } + vm := new(MachineVM) + if len(opts.Name) > 0 { + vm.Name = opts.Name + } + vm.IgnitionFilePath = opts.IgnitionPath + // If no ignitionfilepath was provided, use defaults + if len(vm.IgnitionFilePath) < 1 { + ignitionFile := filepath.Join(vmConfigDir, vm.Name+".ign") + vm.IgnitionFilePath = ignitionFile + } + + // An image was specified + if len(opts.ImagePath) > 0 { + vm.ImagePath = opts.ImagePath + } + + // Assign remote user name. if not provided, use default + vm.RemoteUsername = opts.Username + if len(vm.RemoteUsername) < 1 { + vm.RemoteUsername = defaultRemoteUser + } + + // Add a random port for ssh + port, err := utils.GetRandomPort() + if err != nil { + return nil, err + } + vm.Port = port + + vm.CPUs = opts.CPUS + vm.Memory = opts.Memory + + // Look up the executable + execPath, err := exec.LookPath(QemuCommand) + if err != nil { + return nil, err + } + cmd := append([]string{execPath}) + // Add memory + cmd = append(cmd, []string{"-m", strconv.Itoa(int(vm.Memory))}...) + // Add cpus + cmd = append(cmd, []string{"-smp", strconv.Itoa(int(vm.CPUs))}...) + // Add ignition file + cmd = append(cmd, []string{"-fw_cfg", "name=opt/com.coreos/config,file=" + vm.IgnitionFilePath}...) + // Add qmp socket + monitor, err := NewQMPMonitor("unix", vm.Name, defaultQMPTimeout) + if err != nil { + return nil, err + } + vm.QMPMonitor = monitor + cmd = append(cmd, []string{"-qmp", monitor.Network + ":/" + monitor.Address + ",server,nowait"}...) + + // Add network + cmd = append(cmd, "-nic", "user,model=virtio,hostfwd=tcp::"+strconv.Itoa(vm.Port)+"-:22") + + vm.CmdLine = cmd + return vm, nil +} + +// LoadByName reads a json file that describes a known qemu vm +// and returns a vm instance +func LoadVMByName(name string) (machine.VM, error) { + // TODO need to define an error relating to ErrMachineNotFound + vm := new(MachineVM) + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return nil, err + } + b, err := ioutil.ReadFile(filepath.Join(vmConfigDir, name+".json")) + if err != nil { + return nil, err + } + err = json.Unmarshal(b, vm) + logrus.Debug(vm.CmdLine) + return vm, err +} + +// Init writes the json configuration file to the filesystem for +// other verbs (start, stop) +func (v *MachineVM) Init(opts machine.InitOptions) error { + sshDir := filepath.Join(homedir.Get(), ".ssh") + // GetConfDir creates the directory so no need to check for + // its existence + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return err + } + jsonFile := filepath.Join(vmConfigDir, v.Name) + ".json" + v.IdentityPath = filepath.Join(sshDir, v.Name) + + dd, err := machine.NewFcosDownloader(vmtype, v.Name) + if err != nil { + return err + } + + v.ImagePath = dd.Get().LocalUncompressedFile + if err := dd.DownloadImage(); err != nil { + return err + } + // Add arch specific options including image location + v.CmdLine = append(v.CmdLine, v.addArchOptions()...) + + // Add location of bootable image + v.CmdLine = append(v.CmdLine, "-drive", "if=virtio,file="+v.ImagePath) + // This kind of stinks but no other way around this r/n + uri := machine.SSHRemoteConnection.MakeSSHURL("localhost", "/run/user/1000/podman/podman.sock", strconv.Itoa(v.Port), v.RemoteUsername) + if err := machine.AddConnection(&uri, v.Name, filepath.Join(sshDir, v.Name), opts.IsDefault); err != nil { + return err + } + // Write the JSON file + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonFile, b, 0644); err != nil { + return err + } + key, err := machine.CreateSSHKeys(v.IdentityPath) + if err != nil { + return err + } + // Run arch specific things that need to be done + if err := v.prepare(); err != nil { + return err + } + // Write the ignition file + return machine.NewIgnitionFile(opts.Username, key, v.IgnitionFilePath) +} + +// Start executes the qemu command line and forks it +func (v *MachineVM) Start(name string, _ machine.StartOptions) error { + var ( + err error + ) + attr := new(os.ProcAttr) + files := []*os.File{os.Stdin, os.Stdout, os.Stderr} + attr.Files = files + logrus.Debug(v.CmdLine) + cmd := v.CmdLine + + // Disable graphic window when not in debug mode + // Done in start, so we're not suck with the debug level we used on init + if logrus.GetLevel() != logrus.DebugLevel { + cmd = append(cmd, "-display", "none") + } + + _, err = os.StartProcess(v.CmdLine[0], cmd, attr) + return err +} + +// Stop uses the qmp monitor to call a system_powerdown +func (v *MachineVM) Stop(name string, _ machine.StopOptions) error { + // check if the qmp socket is there. if not, qemu instance is gone + if _, err := os.Stat(v.QMPMonitor.Address); os.IsNotExist(err) { + // Right now it is NOT an error to stop a stopped machine + logrus.Debugf("QMP monitor socket %v does not exist", v.QMPMonitor.Address) + return nil + } + qmpMonitor, err := qmp.NewSocketMonitor(v.QMPMonitor.Network, v.QMPMonitor.Address, v.QMPMonitor.Timeout) + if err != nil { + return err + } + // Simple JSON formation for the QAPI + stopCommand := struct { + Execute string `json:"execute"` + }{ + Execute: "system_powerdown", + } + input, err := json.Marshal(stopCommand) + if err != nil { + return err + } + if err := qmpMonitor.Connect(); err != nil { + return err + } + defer func() { + if err := qmpMonitor.Disconnect(); err != nil { + logrus.Error(err) + } + }() + _, err = qmpMonitor.Run(input) + return err +} + +// NewQMPMonitor creates the monitor subsection of our vm +func NewQMPMonitor(network, name string, timeout time.Duration) (Monitor, error) { + rtDir, err := getSocketDir() + if err != nil { + return Monitor{}, err + } + rtDir = filepath.Join(rtDir, "podman") + if _, err := os.Stat(filepath.Join(rtDir)); os.IsNotExist(err) { + // TODO 0644 is fine on linux but macos is weird + if err := os.MkdirAll(rtDir, 0755); err != nil { + return Monitor{}, err + } + } + if timeout == 0 { + timeout = defaultQMPTimeout + } + monitor := Monitor{ + Network: network, + Address: filepath.Join(rtDir, "qmp_"+name+".sock"), + Timeout: timeout, + } + return monitor, nil +} + +func (v *MachineVM) Remove(name string, opts machine.RemoveOptions) (string, func() error, error) { + var ( + files []string + ) + + // cannot remove a running vm + if v.isRunning() { + return "", nil, errors.Errorf("running vm %q cannot be destroyed", v.Name) + } + + // Collect all the files that need to be destroyed + if !opts.SaveKeys { + files = append(files, v.IdentityPath, v.IdentityPath+".pub") + } + if !opts.SaveIgnition { + files = append(files, v.IgnitionFilePath) + } + if !opts.SaveImage { + files = append(files, v.ImagePath) + } + files = append(files, v.archRemovalFiles()...) + + if err := machine.RemoveConnection(v.Name); err != nil { + logrus.Error(err) + } + vmConfigDir, err := machine.GetConfDir(vmtype) + if err != nil { + return "", nil, err + } + files = append(files, filepath.Join(vmConfigDir, v.Name+".json")) + confirmationMessage := "\nThe following files will be deleted:\n\n" + for _, msg := range files { + confirmationMessage += msg + "\n" + } + confirmationMessage += "\n" + return confirmationMessage, func() error { + for _, f := range files { + if err := os.Remove(f); err != nil { + logrus.Error(err) + } + } + return nil + }, nil +} + +func (v *MachineVM) isRunning() bool { + // Check if qmp socket path exists + if _, err := os.Stat(v.QMPMonitor.Address); os.IsNotExist(err) { + return false + } + // Check if we can dial it + if _, err := qmp.NewSocketMonitor(v.QMPMonitor.Network, v.QMPMonitor.Address, v.QMPMonitor.Timeout); err != nil { + return false + } + return true +} + +// SSH opens an interactive SSH session to the vm specified. +// Added ssh function to VM interface: pkg/machine/config/go : line 58 +func (v *MachineVM) SSH(name string, opts machine.SSHOptions) error { + if !v.isRunning() { + return errors.Errorf("vm %q is not running.", v.Name) + } + + sshDestination := v.RemoteUsername + "@localhost" + port := strconv.Itoa(v.Port) + + args := []string{"-i", v.IdentityPath, "-p", port, sshDestination} + if opts.Execute { + args = append(args, opts.Args...) + } else { + fmt.Printf("Connecting to vm %s. To close connection, use `~.` or `exit`\n", v.Name) + } + + cmd := exec.Command("ssh", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + return cmd.Run() +} diff --git a/pkg/machine/qemu/options_darwin.go b/pkg/machine/qemu/options_darwin.go new file mode 100644 index 000000000..46ccf24cb --- /dev/null +++ b/pkg/machine/qemu/options_darwin.go @@ -0,0 +1,15 @@ +package qemu + +import ( + "os" + + "github.com/pkg/errors" +) + +func getSocketDir() (string, error) { + tmpDir, ok := os.LookupEnv("TMPDIR") + if !ok { + return "", errors.New("unable to resolve TMPDIR") + } + return tmpDir, nil +} diff --git a/pkg/machine/qemu/options_darwin_amd64.go b/pkg/machine/qemu/options_darwin_amd64.go new file mode 100644 index 000000000..69f7982b2 --- /dev/null +++ b/pkg/machine/qemu/options_darwin_amd64.go @@ -0,0 +1,18 @@ +package qemu + +var ( + QemuCommand = "qemu-system-x86_64" +) + +func (v *MachineVM) addArchOptions() []string { + opts := []string{"-cpu", "host"} + return opts +} + +func (v *MachineVM) prepare() error { + return nil +} + +func (v *MachineVM) archRemovalFiles() []string { + return []string{} +} diff --git a/pkg/machine/qemu/options_darwin_arm64.go b/pkg/machine/qemu/options_darwin_arm64.go new file mode 100644 index 000000000..7513b3048 --- /dev/null +++ b/pkg/machine/qemu/options_darwin_arm64.go @@ -0,0 +1,36 @@ +package qemu + +import ( + "os/exec" + "path/filepath" +) + +var ( + QemuCommand = "qemu-system-aarch64" +) + +func (v *MachineVM) addArchOptions() []string { + ovmfDir := getOvmfDir(v.ImagePath, v.Name) + opts := []string{ + "-accel", "hvf", + "-cpu", "cortex-a57", + "-M", "virt,highmem=off", + "-drive", "file=/usr/local/share/qemu/edk2-aarch64-code.fd,if=pflash,format=raw,readonly=on", + "-drive", "file=" + ovmfDir + ",if=pflash,format=raw"} + return opts +} + +func (v *MachineVM) prepare() error { + ovmfDir := getOvmfDir(v.ImagePath, v.Name) + cmd := []string{"dd", "if=/dev/zero", "conv=sync", "bs=1m", "count=64", "of=" + ovmfDir} + return exec.Command(cmd[0], cmd[1:]...).Run() +} + +func (v *MachineVM) archRemovalFiles() []string { + ovmDir := getOvmfDir(v.ImagePath, v.Name) + return []string{ovmDir} +} + +func getOvmfDir(imagePath, vmName string) string { + return filepath.Join(filepath.Dir(imagePath), vmName+"_ovmf_vars.fd") +} diff --git a/pkg/machine/qemu/options_linux.go b/pkg/machine/qemu/options_linux.go new file mode 100644 index 000000000..0a2e40d8f --- /dev/null +++ b/pkg/machine/qemu/options_linux.go @@ -0,0 +1,7 @@ +package qemu + +import "github.com/containers/podman/v3/pkg/util" + +func getSocketDir() (string, error) { + return util.GetRuntimeDir() +} diff --git a/pkg/machine/qemu/options_linux_amd64.go b/pkg/machine/qemu/options_linux_amd64.go new file mode 100644 index 000000000..cc0a4bab2 --- /dev/null +++ b/pkg/machine/qemu/options_linux_amd64.go @@ -0,0 +1,18 @@ +package qemu + +var ( + QemuCommand = "qemu-kvm" +) + +func (v *MachineVM) addArchOptions() []string { + opts := []string{"-cpu", "host"} + return opts +} + +func (v *MachineVM) prepare() error { + return nil +} + +func (v *MachineVM) archRemovalFiles() []string { + return []string{} +} diff --git a/pkg/specgen/generate/ports.go b/pkg/specgen/generate/ports.go index 6cf83ed81..678e36a70 100644 --- a/pkg/specgen/generate/ports.go +++ b/pkg/specgen/generate/ports.go @@ -6,6 +6,8 @@ import ( "strconv" "strings" + "github.com/containers/podman/v3/utils" + "github.com/containers/podman/v3/libpod/image" "github.com/containers/podman/v3/pkg/specgen" "github.com/cri-o/ocicni/pkg/ocicni" @@ -218,7 +220,7 @@ func parsePortMapping(portMappings []specgen.PortMapping) ([]ocicni.PortMapping, // Only get a random candidate for single entries or the start // of a range. Otherwise we just increment the candidate. if !tmp.isInRange || tmp.startOfRange { - candidate, err = getRandomPort() + candidate, err = utils.GetRandomPort() if err != nil { return nil, nil, nil, errors.Wrapf(err, "error getting candidate host port for container port %d", p.ContainerPort) } @@ -344,7 +346,7 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, img *imag for hostPort == 0 && tries > 0 { // We can't select a specific protocol, which is // unfortunate for the UDP case. - candidate, err := getRandomPort() + candidate, err := utils.GetRandomPort() if err != nil { return nil, err } @@ -419,21 +421,3 @@ func checkProtocol(protocol string, allowSCTP bool) ([]string, error) { return finalProto, nil } - -// Find a random, open port on the host -func getRandomPort() (int, error) { - l, err := net.Listen("tcp", ":0") - if err != nil { - return 0, errors.Wrapf(err, "unable to get free TCP port") - } - defer l.Close() - _, randomPort, err := net.SplitHostPort(l.Addr().String()) - if err != nil { - return 0, errors.Wrapf(err, "unable to determine free port") - } - rp, err := strconv.Atoi(randomPort) - if err != nil { - return 0, errors.Wrapf(err, "unable to convert random port to int") - } - return rp, nil -} diff --git a/pkg/specgen/generate/security.go b/pkg/specgen/generate/security.go index 56aac8bfd..e0e4a47a4 100644 --- a/pkg/specgen/generate/security.go +++ b/pkg/specgen/generate/security.go @@ -89,12 +89,28 @@ func securityConfigureGenerator(s *specgen.SpecGenerator, g *generate.Generator, // NOTE: Must happen before SECCOMP if s.Privileged { g.SetupPrivileged(true) - caplist = capabilities.AllCapabilities() + caplist, err = capabilities.BoundingSet() + if err != nil { + return err + } } else { - caplist, err = capabilities.MergeCapabilities(rtc.Containers.DefaultCapabilities, s.CapAdd, s.CapDrop) + mergedCaps, err := capabilities.MergeCapabilities(rtc.Containers.DefaultCapabilities, s.CapAdd, s.CapDrop) + if err != nil { + return err + } + boundingSet, err := capabilities.BoundingSet() if err != nil { return err } + boundingCaps := make(map[string]interface{}) + for _, b := range boundingSet { + boundingCaps[b] = b + } + for _, c := range mergedCaps { + if _, ok := boundingCaps[c]; ok { + caplist = append(caplist, c) + } + } privCapsRequired := []string{} @@ -139,10 +155,24 @@ func securityConfigureGenerator(s *specgen.SpecGenerator, g *generate.Generator, configSpec.Process.Capabilities.Permitted = caplist configSpec.Process.Capabilities.Inheritable = caplist } else { - userCaps, err := capabilities.MergeCapabilities(nil, s.CapAdd, nil) + mergedCaps, err := capabilities.MergeCapabilities(nil, s.CapAdd, nil) if err != nil { return errors.Wrapf(err, "capabilities requested by user are not valid: %q", strings.Join(s.CapAdd, ",")) } + boundingSet, err := capabilities.BoundingSet() + if err != nil { + return err + } + boundingCaps := make(map[string]interface{}) + for _, b := range boundingSet { + boundingCaps[b] = b + } + var userCaps []string + for _, c := range mergedCaps { + if _, ok := boundingCaps[c]; ok { + userCaps = append(userCaps, c) + } + } configSpec.Process.Capabilities.Effective = userCaps configSpec.Process.Capabilities.Permitted = userCaps configSpec.Process.Capabilities.Inheritable = userCaps diff --git a/pkg/specgen/namespaces.go b/pkg/specgen/namespaces.go index fb7d65da4..f665fc0be 100644 --- a/pkg/specgen/namespaces.go +++ b/pkg/specgen/namespaces.go @@ -54,7 +54,7 @@ const ( // Namespace describes the namespace type Namespace struct { NSMode NamespaceMode `json:"nsmode,omitempty"` - Value string `json:"string,omitempty"` + Value string `json:"value,omitempty"` } // IsDefault returns whether the namespace is set to the default setting (which diff --git a/pkg/util/filters.go b/pkg/util/filters.go new file mode 100644 index 000000000..43bf646f1 --- /dev/null +++ b/pkg/util/filters.go @@ -0,0 +1,116 @@ +package util + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/containers/podman/v3/pkg/timetype" + "github.com/pkg/errors" +) + +// ComputeUntilTimestamp extracts until timestamp from filters +func ComputeUntilTimestamp(filterValues []string) (time.Time, error) { + invalid := time.Time{} + if len(filterValues) != 1 { + return invalid, errors.Errorf("specify exactly one timestamp for until") + } + ts, err := timetype.GetTimestamp(filterValues[0], time.Now()) + if err != nil { + return invalid, err + } + seconds, nanoseconds, err := timetype.ParseTimestamps(ts, 0) + if err != nil { + return invalid, err + } + return time.Unix(seconds, nanoseconds), nil +} + +// filtersFromRequests extracts the "filters" parameter from the specified +// http.Request. The parameter can either be a `map[string][]string` as done +// in new versions of Docker and libpod, or a `map[string]map[string]bool` as +// done in older versions of Docker. We have to do a bit of Yoga to support +// both - just as Docker does as well. +// +// Please refer to https://github.com/containers/podman/issues/6899 for some +// background. +func FiltersFromRequest(r *http.Request) ([]string, error) { + var ( + compatFilters map[string]map[string]bool + filters map[string][]string + libpodFilters []string + raw []byte + ) + + if _, found := r.URL.Query()["filters"]; found { + raw = []byte(r.Form.Get("filters")) + } else if _, found := r.URL.Query()["Filters"]; found { + raw = []byte(r.Form.Get("Filters")) + } else { + return []string{}, nil + } + + // Backwards compat with older versions of Docker. + if err := json.Unmarshal(raw, &compatFilters); err == nil { + for filterKey, filterMap := range compatFilters { + for filterValue, toAdd := range filterMap { + if toAdd { + libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", filterKey, filterValue)) + } + } + } + return libpodFilters, nil + } + + if err := json.Unmarshal(raw, &filters); err != nil { + return nil, err + } + + for filterKey, filterSlice := range filters { + for _, filterValue := range filterSlice { + libpodFilters = append(libpodFilters, fmt.Sprintf("%s=%s", filterKey, filterValue)) + } + } + + return libpodFilters, nil +} + +// PrepareFilters prepares a *map[string][]string of filters to be later searched +// in lipod and compat API to get desired filters +func PrepareFilters(r *http.Request) (*map[string][]string, error) { + filtersList, err := FiltersFromRequest(r) + if err != nil { + return nil, err + } + filterMap := map[string][]string{} + for _, filter := range filtersList { + split := strings.SplitN(filter, "=", 2) + if len(split) > 1 { + filterMap[split[0]] = append(filterMap[split[0]], split[1]) + } + } + return &filterMap, nil +} + +// MatchLabelFilters matches labels and returs true if they are valid +func MatchLabelFilters(filterValues []string, labels map[string]string) bool { +outer: + for _, filterValue := range filterValues { + filterArray := strings.SplitN(filterValue, "=", 2) + filterKey := filterArray[0] + if len(filterArray) > 1 { + filterValue = filterArray[1] + } else { + filterValue = "" + } + for labelKey, labelValue := range labels { + if labelKey == filterKey && (filterValue == "" || labelValue == filterValue) { + continue outer + } + } + return false + } + return true +} diff --git a/pkg/util/filters_test.go b/pkg/util/filters_test.go new file mode 100644 index 000000000..47259013e --- /dev/null +++ b/pkg/util/filters_test.go @@ -0,0 +1,113 @@ +package util + +import ( + "testing" +) + +func TestMatchLabelFilters(t *testing.T) { + testLabels := map[string]string{ + "label1": "", + "label2": "test", + "label3": "", + } + type args struct { + filterValues []string + labels map[string]string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Match when all filters the same as labels", + args: args{ + filterValues: []string{"label1", "label3", "label2=test"}, + labels: testLabels, + }, + want: true, + }, + { + name: "Match when filter value not provided in args", + args: args{ + filterValues: []string{"label2"}, + labels: testLabels, + }, + want: true, + }, + { + name: "Match when no filter value is given", + args: args{ + filterValues: []string{"label2="}, + labels: testLabels, + }, + want: true, + }, + { + name: "Do not match when filter value differs", + args: args{ + filterValues: []string{"label2=differs"}, + labels: testLabels, + }, + want: false, + }, + { + name: "Do not match when filter value not listed in labels", + args: args{ + filterValues: []string{"label1=xyz"}, + labels: testLabels, + }, + want: false, + }, + { + name: "Do not match when one from many not ok", + args: args{ + filterValues: []string{"label1=xyz", "invalid=valid"}, + labels: testLabels, + }, + want: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + if got := MatchLabelFilters(tt.args.filterValues, tt.args.labels); got != tt.want { + t.Errorf("MatchLabelFilters() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestComputeUntilTimestamp(t *testing.T) { + tests := []struct { + name string + args []string + wantErr bool + }{ + { + name: "Return error when more values in list", + args: []string{"5h", "6s"}, + wantErr: true, + }, + { + name: "Return error when invalid time", + args: []string{"invalidTime"}, + wantErr: true, + }, + { + name: "Do not return error when correct time format supplied", + args: []string{"44m"}, + wantErr: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + _, err := ComputeUntilTimestamp(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("ComputeUntilTimestamp() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} |