diff options
64 files changed, 1244 insertions, 677 deletions
diff --git a/cmd/podman/containers/rm.go b/cmd/podman/containers/rm.go index 70cb76693..cede0ba14 100644 --- a/cmd/podman/containers/rm.go +++ b/cmd/podman/containers/rm.go @@ -61,7 +61,8 @@ func rmFlags(cmd *cobra.Command) { flags.BoolVarP(&rmOptions.All, "all", "a", false, "Remove all containers") flags.BoolVarP(&rmOptions.Ignore, "ignore", "i", false, "Ignore errors when a specified container is missing") - flags.BoolVarP(&rmOptions.Force, "force", "f", false, "Force removal of a running or unusable container. The default is false") + flags.BoolVarP(&rmOptions.Force, "force", "f", false, "Force removal of a running or unusable container") + flags.BoolVar(&rmOptions.Depend, "depend", false, "Remove container and all containers that depend on the selected container") timeFlagName := "time" flags.UintVarP(&stopTimeout, timeFlagName, "t", containerConfig.Engine.StopTimeout, "Seconds to wait for stop before killing the container") _ = cmd.RegisterFlagCompletionFunc(timeFlagName, completion.AutocompleteNone) diff --git a/cmd/podman/images/scp.go b/cmd/podman/images/scp.go index f02a3c15e..fb20d9417 100644 --- a/cmd/podman/images/scp.go +++ b/cmd/podman/images/scp.go @@ -46,6 +46,7 @@ var ( var ( parentFlags []string + quiet bool source entities.ImageScpOptions dest entities.ImageScpOptions sshInfo entities.ImageScpConnections @@ -61,7 +62,7 @@ func init() { func scpFlags(cmd *cobra.Command) { flags := cmd.Flags() - flags.BoolVarP(&source.Quiet, "quiet", "q", false, "Suppress the output") + flags.BoolVarP(&quiet, "quiet", "q", false, "Suppress the output") } func scp(cmd *cobra.Command, args []string) (finalErr error) { @@ -139,6 +140,7 @@ func scp(cmd *cobra.Command, args []string) (finalErr error) { } } + source.Quiet = quiet source.File = f.Name() // after parsing the arguments, set the file for the save/load dest.File = source.File if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors @@ -203,15 +205,6 @@ func scp(cmd *cobra.Command, args []string) (finalErr error) { } } - src, err := json.MarshalIndent(source, "", " ") - if err != nil { - return err - } - dst, err := json.MarshalIndent(dest, "", " ") - if err != nil { - return err - } - fmt.Printf("SOURCE: %s\nDEST: %s\n", string(src), string(dst)) return nil } diff --git a/cmd/podman/machine/list.go b/cmd/podman/machine/list.go index ed43b42df..3d8def0db 100644 --- a/cmd/podman/machine/list.go +++ b/cmd/podman/machine/list.go @@ -44,16 +44,19 @@ type listFlagType struct { } type machineReporter struct { - Name string - Default bool - Created string - Running bool - LastUp string - Stream string - VMType string - CPUs uint64 - Memory string - DiskSize string + Name string + Default bool + Created string + Running bool + LastUp string + Stream string + VMType string + CPUs uint64 + Memory string + DiskSize string + Port int + RemoteUsername string + IdentityPath string } func init() { @@ -190,6 +193,9 @@ func toMachineFormat(vms []*machine.ListResponse) ([]*machineReporter, error) { response.CPUs = vm.CPUs response.Memory = strUint(vm.Memory) response.DiskSize = strUint(vm.DiskSize) + response.Port = vm.Port + response.RemoteUsername = vm.RemoteUsername + response.IdentityPath = vm.IdentityPath machineResponses = append(machineResponses, response) } diff --git a/cmd/podman/root.go b/cmd/podman/root.go index bccc559ce..1de937ca5 100644 --- a/cmd/podman/root.go +++ b/cmd/podman/root.go @@ -114,6 +114,10 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { } cfg := registry.PodmanConfig() + if cfg.NoOut { + null, _ := os.Open(os.DevNull) + os.Stdout = null + } // Currently it is only possible to restore a container with the same runtime // as used for checkpointing. It should be possible to make crun and runc @@ -137,7 +141,7 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { runtimeFlag := cmd.Root().Flags().Lookup("runtime") if runtimeFlag == nil { return errors.Errorf( - "Unexcpected error setting runtime to '%s' for restore", + "setting runtime to '%s' for restore", *runtime, ) } @@ -217,7 +221,7 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { context := cmd.Root().LocalFlags().Lookup("context") if context.Value.String() != "default" { - return errors.New("Podman does not support swarm, the only --context value allowed is \"default\"") + return errors.New("podman does not support swarm, the only --context value allowed is \"default\"") } if !registry.IsRemote() { if cmd.Flag("cpu-profile").Changed { @@ -343,6 +347,7 @@ func rootFlags(cmd *cobra.Command, opts *entities.PodmanConfig) { lFlags.StringVar(&opts.Identity, identityFlagName, ident, "path to SSH identity file, (CONTAINER_SSHKEY)") _ = cmd.RegisterFlagCompletionFunc(identityFlagName, completion.AutocompleteDefault) + lFlags.BoolVar(&opts.NoOut, "noout", false, "do not output to stdout") lFlags.BoolVarP(&opts.Remote, "remote", "r", registry.IsRemote(), "Access remote Podman service") pFlags := cmd.PersistentFlags() if registry.IsRemote() { diff --git a/cmd/podman/utils/error.go b/cmd/podman/utils/error.go index aab1da675..b3b54876f 100644 --- a/cmd/podman/utils/error.go +++ b/cmd/podman/utils/error.go @@ -27,7 +27,7 @@ func (o OutputErrors) PrintErrors() (lastError error) { instead returns a message and we cast it to a new error. Following function performs parsing on build error and returns - exit status which was exepected for this current build + exit status which was expected for this current build */ func ExitCodeFromBuildError(errorMsg string) (int, error) { if strings.Contains(errorMsg, "exit status") { diff --git a/cmd/winpath/main.go b/cmd/winpath/main.go index 494d1cf3c..6fbe72837 100644 --- a/cmd/winpath/main.go +++ b/cmd/winpath/main.go @@ -114,7 +114,7 @@ func addPathToRegistry(dir string) error { return err } -// Removes all occurences of a directory path from the Windows path stored in the registry +// Removes all occurrences of a directory path from the Windows path stored in the registry func removePathFromRegistry(path string) error { k, err := registry.OpenKey(registry.CURRENT_USER, Environment, registry.READ|registry.WRITE) if err != nil { @@ -155,7 +155,7 @@ func removePathFromRegistry(path string) error { return err } -// Sends a notification message to all top level windows informing them the environmental setings have changed. +// Sends a notification message to all top level windows informing them the environmental settings have changed. // Applications such as the Windows command prompt and powershell will know to stop caching stale values on // subsequent restarts. Since applications block the sender when receiving a message, we set a 3 second timeout func broadcastEnvironmentChange() { diff --git a/docs/source/markdown/podman-machine-list.1.md b/docs/source/markdown/podman-machine-list.1.md index b2596ac59..d68b8b1ca 100644 --- a/docs/source/markdown/podman-machine-list.1.md +++ b/docs/source/markdown/podman-machine-list.1.md @@ -37,6 +37,9 @@ Valid placeholders for the Go template are listed below: | .Running | Is machine running | | .Stream | Stream name | | .VMType | VM type | +| .Port | SSH Port to use to connect to VM| +| .RemoteUsername | VM Username for rootless Podman | +| .IdentityPath | Path to ssh identify file | #### **--help** diff --git a/docs/source/markdown/podman-rm.1.md b/docs/source/markdown/podman-rm.1.md index c599c8687..f3807d2f7 100644 --- a/docs/source/markdown/podman-rm.1.md +++ b/docs/source/markdown/podman-rm.1.md @@ -18,6 +18,10 @@ Running or unusable containers will not be removed without the **-f** option. Remove all containers. Can be used in conjunction with **-f** as well. +#### **--depend** + +Remove selected container and recursively remove all containers that depend on it. + #### **--cidfile** Read container ID from the specified file and remove the container. Can be specified multiple times. @@ -56,6 +60,11 @@ Remove a container by its name *mywebserver* $ podman rm mywebserver ``` +Remove a *mywebserver* container and all of the containers that depend on it +``` +$ podman rm --depend mywebserver +``` + Remove several containers by name and container id. ``` $ podman rm mywebserver myflaskserver 860a4b23 diff --git a/docs/source/markdown/podman.1.md b/docs/source/markdown/podman.1.md index daa8212c5..9f85ebda3 100644 --- a/docs/source/markdown/podman.1.md +++ b/docs/source/markdown/podman.1.md @@ -92,6 +92,11 @@ When namespace is set, created containers and pods will join the given namespace #### **--network-cmd-path**=*path* Path to the command binary to use for setting up a network. It is currently only used for setting up a slirp4netns network. If "" is used then the binary is looked up using the $PATH environment variable. +#### **--noout** + +Redirect stdout to /dev/null. This command will prevent all stdout from the Podman command. The **--noout** option will not block stderr or stdout from containers. + + #### **--remote**, **-r** When true, access to the Podman service will be remote. Defaults to false. Settings can be modified in the containers.conf file. If the CONTAINER_HOST @@ -3,7 +3,7 @@ module github.com/containers/podman/v3 go 1.16 require ( - github.com/BurntSushi/toml v0.4.1 + github.com/BurntSushi/toml v1.0.0 github.com/blang/semver v3.5.1+incompatible github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37 github.com/checkpoint-restore/checkpointctl v0.0.0-20211204171957-54b4ebfdb681 @@ -12,7 +12,7 @@ require ( github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.0.1 github.com/containers/buildah v1.23.1 - github.com/containers/common v0.46.1-0.20220110152253-5476e2b8dc46 + github.com/containers/common v0.46.1-0.20220112112017-31e8cc4aeeab github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.17.1-0.20220106205022-73f80d60f0e1 github.com/containers/ocicrypt v1.1.2 @@ -65,8 +65,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -283,8 +284,8 @@ github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNB github.com/containers/buildah v1.23.1 h1:Tpc9DsRuU+0Oofewpxb6OJVNQjCu7yloN/obUqzfDTY= github.com/containers/buildah v1.23.1/go.mod h1:4WnrN0yrA7ab0ppgunixu2WM1rlD2rG8QLJAKbEkZlQ= github.com/containers/common v0.44.2/go.mod h1:7sdP4vmI5Bm6FPFxb3lvAh1Iktb6tiO1MzjUzhxdoGo= -github.com/containers/common v0.46.1-0.20220110152253-5476e2b8dc46 h1:LHeJjs8IJ4d9k8bCNs6L8lesi10Zk0LNnZ3fyxL6uXk= -github.com/containers/common v0.46.1-0.20220110152253-5476e2b8dc46/go.mod h1:hXUU9gtA8V9dSLHhizp/k/s0ZXBzrnUSScUfrsw8z2Y= +github.com/containers/common v0.46.1-0.20220112112017-31e8cc4aeeab h1:Pf1kwI8sZPiPMuen619noeltwtB2cIFC1iY42fE87AY= +github.com/containers/common v0.46.1-0.20220112112017-31e8cc4aeeab/go.mod h1:hXUU9gtA8V9dSLHhizp/k/s0ZXBzrnUSScUfrsw8z2Y= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.16.0/go.mod h1:XgTpfAPLRGOd1XYyCU5cISFr777bLmOerCSpt/v7+Q4= diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 7745646b6..28d961e4b 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -2221,33 +2221,50 @@ func (c *Container) getHosts() string { depCtr = c } + // getLocalIP returns the non loopback local IP of the host + getLocalIP := func() string { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "" + } + for _, address := range addrs { + // check the address type and if it is not a loopback the display it + if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String() + } + } + } + return "" + } + if depCtr != nil { - for _, status := range depCtr.getNetworkStatus() { + host := "" + outer: + for net, status := range depCtr.getNetworkStatus() { + network, err := c.runtime.network.NetworkInspect(net) + // only add the host entry for bridge networks + // ip/macvlan gateway is normally not on the host + if err != nil || network.Driver != types.BridgeNetworkDriver { + continue + } for _, netInt := range status.Interfaces { for _, netAddress := range netInt.Subnets { if netAddress.Gateway != nil { - hosts += fmt.Sprintf("%s host.containers.internal\n", netAddress.Gateway.String()) + host = fmt.Sprintf("%s host.containers.internal\n", netAddress.Gateway.String()) + break outer } } } } - } else if c.config.NetMode.IsSlirp4netns() { - // getLocalIP returns the non loopback local IP of the host - getLocalIP := func() string { - addrs, err := net.InterfaceAddrs() - if err != nil { - return "" - } - for _, address := range addrs { - // check the address type and if it is not a loopback the display it - if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { - if ipnet.IP.To4() != nil { - return ipnet.IP.String() - } - } + // if no bridge gw was found try to use a local ip + if host == "" { + if ip := getLocalIP(); ip != "" { + host = fmt.Sprintf("%s\t%s\n", ip, "host.containers.internal") } - return "" } + hosts += host + } else if c.config.NetMode.IsSlirp4netns() { if ip := getLocalIP(); ip != "" { hosts += fmt.Sprintf("%s\t%s\n", ip, "host.containers.internal") } diff --git a/libpod/runtime.go b/libpod/runtime.go index 9794b3605..3297b1ddd 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -170,7 +170,6 @@ func NewRuntime(ctx context.Context, options ...RuntimeOption) (*Runtime, error) if err != nil { return nil, err } - conf.CheckCgroupsAndAdjustConfig() return newRuntimeFromConfig(ctx, conf, options...) } @@ -228,6 +227,8 @@ func newRuntimeFromConfig(ctx context.Context, conf *config.Config, options ...R return nil, err } + runtime.config.CheckCgroupsAndAdjustConfig() + return runtime, nil } diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 59a1fd153..53ccb9139 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -429,7 +429,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai }() ctr.config.SecretsPath = filepath.Join(ctr.config.StaticDir, "secrets") - err = os.MkdirAll(ctr.config.SecretsPath, 0644) + err = os.MkdirAll(ctr.config.SecretsPath, 0755) if err != nil { return nil, err } @@ -915,6 +915,37 @@ func (r *Runtime) evictContainer(ctx context.Context, idOrName string, removeVol return id, cleanupErr } +// RemoveDepend removes all dependencies for a container +func (r *Runtime) RemoveDepend(ctx context.Context, rmCtr *Container, force bool, removeVolume bool, timeout *uint) ([]*reports.RmReport, error) { + rmReports := make([]*reports.RmReport, 0) + deps, err := r.state.ContainerInUse(rmCtr) + if err != nil { + if err == define.ErrCtrRemoved { + return rmReports, nil + } + return rmReports, err + } + for _, cid := range deps { + ctr, err := r.state.Container(cid) + if err != nil { + if err == define.ErrNoSuchCtr { + continue + } + return rmReports, err + } + + reports, err := r.RemoveDepend(ctx, ctr, force, removeVolume, timeout) + if err != nil { + return rmReports, err + } + rmReports = append(rmReports, reports...) + } + report := reports.RmReport{Id: rmCtr.ID()} + report.Err = r.removeContainer(ctx, rmCtr, force, removeVolume, false, timeout) + + return append(rmReports, &report), nil +} + // GetContainer retrieves a container by its ID func (r *Runtime) GetContainer(id string) (*Container, error) { r.lock.RLock() diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index ad341c3ab..4539199d3 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -36,6 +36,7 @@ func RemoveContainer(w http.ResponseWriter, r *http.Request) { query := struct { Force bool `schema:"force"` Ignore bool `schema:"ignore"` + Depend bool `schema:"depend"` Link bool `schema:"link"` Timeout *uint `schema:"timeout"` DockerVolumes bool `schema:"v"` @@ -57,6 +58,7 @@ func RemoveContainer(w http.ResponseWriter, r *http.Request) { if utils.IsLibpodRequest(r) { options.Volumes = query.LibpodVolumes options.Timeout = query.Timeout + options.Depend = query.Depend } else { if query.Link { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, @@ -71,7 +73,7 @@ func RemoveContainer(w http.ResponseWriter, r *http.Request) { // code. containerEngine := abi.ContainerEngine{Libpod: runtime} name := utils.GetName(r) - report, err := containerEngine.ContainerRm(r.Context(), []string{name}, options) + reports, err := containerEngine.ContainerRm(r.Context(), []string{name}, options) if err != nil { if errors.Cause(err) == define.ErrNoSuchCtr { utils.ContainerNotFound(w, name, err) @@ -81,8 +83,8 @@ func RemoveContainer(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } - if len(report) > 0 && report[0].Err != nil { - err = report[0].Err + if len(reports) > 0 && reports[0].Err != nil { + err = reports[0].Err if errors.Cause(err) == define.ErrNoSuchCtr { utils.ContainerNotFound(w, name, err) return @@ -90,7 +92,10 @@ func RemoveContainer(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } - + if utils.IsLibpodRequest(r) { + utils.WriteResponse(w, http.StatusOK, reports) + return + } utils.WriteResponse(w, http.StatusNoContent, nil) } diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 0fcac5330..2d296b5ce 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -138,7 +138,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { // if layers field not set assume its not from a valid podman-client // could be a docker client, set `layers=true` since that is the default - // expected behviour + // expected behaviour if !utils.IsLibpodRequest(r) { if _, found := r.URL.Query()["layers"]; !found { query.Layers = true diff --git a/pkg/api/handlers/swagger/swagger.go b/pkg/api/handlers/swagger/swagger.go index 9844839b7..7868ff206 100644 --- a/pkg/api/handlers/swagger/swagger.go +++ b/pkg/api/handlers/swagger/swagger.go @@ -111,6 +111,13 @@ type swagLibpodInspectImageResponse struct { } } +// Rm containers +// swagger:response DocsLibpodContainerRmReport +type swagLibpodContainerRmReport struct { + // in: body + Body []handlers.LibpodContainersRmReport +} + // Prune containers // swagger:response DocsContainerPruneReport type swagContainerPruneReport struct { diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index f850db3d8..588758b2c 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -53,6 +53,17 @@ type LibpodContainersPruneReport struct { PruneError string `json:"Err,omitempty"` } +type LibpodContainersRmReport struct { + ID string `json:"Id"` + // Error which occurred during Rm operation (if any). + // This field is optional and may be omitted if no error occurred. + // + // Extensions: + // x-omitempty: true + // x-nullable: true + RmError string `json:"Err,omitempty"` +} + type Info struct { docker.Info BuildahVersion string diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go index 601e1251b..4d19c04d4 100644 --- a/pkg/api/server/register_containers.go +++ b/pkg/api/server/register_containers.go @@ -817,9 +817,22 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // required: true // description: the name or ID of the container // - in: query + // name: depend + // type: boolean + // description: additionally remove containers that depend on the container to be removed + // - in: query // name: force // type: boolean - // description: need something + // description: force stop container if running + // - in: query + // name: ignore + // type: boolean + // description: ignore errors when the container to be removed does not existxo + // - in: query + // name: timeout + // type: integer + // default: 10 + // description: number of seconds to wait before killing container when force removing // - in: query // name: v // type: boolean @@ -827,6 +840,8 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // produces: // - application/json // responses: + // 200: + // $ref: "#/responses/DocsLibpodContainerRmReport" // 204: // description: no error // 400: diff --git a/pkg/bindings/containers/containers.go b/pkg/bindings/containers/containers.go index 14a173025..0148e62cb 100644 --- a/pkg/bindings/containers/containers.go +++ b/pkg/bindings/containers/containers.go @@ -78,25 +78,26 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*reports.PruneReport, // The volumes bool dictates that a container's volumes should also be removed. // The All option indicates that all containers should be removed // The Ignore option indicates that if a container did not exist, ignore the error -func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) error { +func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) ([]*reports.RmReport, error) { if options == nil { options = new(RemoveOptions) } + var reports []*reports.RmReport conn, err := bindings.GetClient(ctx) if err != nil { - return err + return reports, err } params, err := options.ToParams() if err != nil { - return err + return reports, err } response, err := conn.DoRequest(ctx, nil, http.MethodDelete, "/containers/%s", params, nil, nameOrID) if err != nil { - return err + return reports, err } defer response.Body.Close() - return response.Process(nil) + return reports, response.Process(&reports) } // Inspect returns low level information about a Container. The nameOrID can be a container name diff --git a/pkg/bindings/containers/types.go b/pkg/bindings/containers/types.go index 81a53a549..db3eb3e1b 100644 --- a/pkg/bindings/containers/types.go +++ b/pkg/bindings/containers/types.go @@ -138,6 +138,7 @@ type PruneOptions struct { //go:generate go run ../generator/generator.go RemoveOptions // RemoveOptions are optional options for removing containers type RemoveOptions struct { + Depend *bool Ignore *bool Force *bool Volumes *bool diff --git a/pkg/bindings/containers/types_remove_options.go b/pkg/bindings/containers/types_remove_options.go index 1e52e819d..7fa198d2f 100644 --- a/pkg/bindings/containers/types_remove_options.go +++ b/pkg/bindings/containers/types_remove_options.go @@ -17,6 +17,21 @@ func (o *RemoveOptions) ToParams() (url.Values, error) { return util.ToParams(o) } +// WithDepend set field Depend to given value +func (o *RemoveOptions) WithDepend(value bool) *RemoveOptions { + o.Depend = &value + return o +} + +// GetDepend returns value of field Depend +func (o *RemoveOptions) GetDepend() bool { + if o.Depend == nil { + var z bool + return z + } + return *o.Depend +} + // WithIgnore set field Ignore to given value func (o *RemoveOptions) WithIgnore(value bool) *RemoveOptions { o.Ignore = &value diff --git a/pkg/bindings/test/containers_test.go b/pkg/bindings/test/containers_test.go index b6c06756b..cab032a40 100644 --- a/pkg/bindings/test/containers_test.go +++ b/pkg/bindings/test/containers_test.go @@ -175,7 +175,7 @@ var _ = Describe("Podman containers ", func() { Expect(err).To(BeNil()) err = containers.Pause(bt.conn, cid, nil) Expect(err).To(BeNil()) - err = containers.Remove(bt.conn, cid, nil) + _, err = containers.Remove(bt.conn, cid, nil) Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -188,8 +188,10 @@ var _ = Describe("Podman containers ", func() { Expect(err).To(BeNil()) err = containers.Pause(bt.conn, cid, nil) Expect(err).To(BeNil()) - err = containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true)) + rmResponse, err := containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true)) Expect(err).To(BeNil()) + Expect(len(reports.RmReportsErrs(rmResponse))).To(Equal(0)) + Expect(len(reports.RmReportsIds(rmResponse))).To(Equal(1)) }) It("podman stop a paused container by name", func() { @@ -669,7 +671,8 @@ var _ = Describe("Podman containers ", func() { }) It("podman remove bogus container", func() { - err = containers.Remove(bt.conn, "foobar", nil) + _, err := containers.Remove(bt.conn, "foobar", nil) + Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusNotFound)) }) @@ -679,7 +682,7 @@ var _ = Describe("Podman containers ", func() { _, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) // Removing running container should fail - err = containers.Remove(bt.conn, name, nil) + _, err = containers.Remove(bt.conn, name, nil) Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -690,7 +693,7 @@ var _ = Describe("Podman containers ", func() { cid, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) // Removing running container should fail - err = containers.Remove(bt.conn, cid, nil) + _, err = containers.Remove(bt.conn, cid, nil) Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -700,22 +703,22 @@ var _ = Describe("Podman containers ", func() { var name = "top" _, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) - // Removing running container should fail - err = containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithForce(true)) + // Removing running container should succeed + rmResponse, err := containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithForce(true)) Expect(err).To(BeNil()) - //code, _ := bindings.CheckResponseCode(err) - //Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) + Expect(len(reports.RmReportsErrs(rmResponse))).To(Equal(0)) + Expect(len(reports.RmReportsIds(rmResponse))).To(Equal(1)) }) It("podman forcibly remove running container by ID", func() { var name = "top" cid, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) - // Removing running container should fail - err = containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true)) + // Forcably Removing running container should succeed + rmResponse, err := containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true)) Expect(err).To(BeNil()) - //code, _ := bindings.CheckResponseCode(err) - //Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) + Expect(len(reports.RmReportsErrs(rmResponse))).To(Equal(0)) + Expect(len(reports.RmReportsIds(rmResponse))).To(Equal(1)) }) It("podman remove running container and volume by name", func() { @@ -723,7 +726,7 @@ var _ = Describe("Podman containers ", func() { _, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) // Removing running container should fail - err = containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithVolumes(true)) + _, err = containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithVolumes(true)) Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -734,7 +737,7 @@ var _ = Describe("Podman containers ", func() { cid, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) // Removing running container should fail - err = containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithVolumes(true)) + _, err = containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithVolumes(true)) Expect(err).ToNot(BeNil()) code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -744,11 +747,11 @@ var _ = Describe("Podman containers ", func() { var name = "top" _, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) - // Removing running container should fail - err = containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithVolumes(true).WithForce(true)) + // Forcibly Removing running container should succeed + rmResponse, err := containers.Remove(bt.conn, name, new(containers.RemoveOptions).WithVolumes(true).WithForce(true)) Expect(err).To(BeNil()) - //code, _ := bindings.CheckResponseCode(err) - //Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) + Expect(len(reports.RmReportsErrs(rmResponse))).To(Equal(0)) + Expect(len(reports.RmReportsIds(rmResponse))).To(Equal(1)) }) It("podman forcibly remove running container and volume by ID", func() { @@ -756,10 +759,10 @@ var _ = Describe("Podman containers ", func() { cid, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) // Removing running container should fail - err = containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true).WithVolumes(true)) + rmResponse, err := containers.Remove(bt.conn, cid, new(containers.RemoveOptions).WithForce(true).WithVolumes(true)) Expect(err).To(BeNil()) - //code, _ := bindings.CheckResponseCode(err) - //Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) + Expect(len(reports.RmReportsErrs(rmResponse))).To(Equal(0)) + Expect(len(reports.RmReportsIds(rmResponse))).To(Equal(1)) }) It("List containers with filters", func() { diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index ae441b7f3..e3f8f1b7c 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -129,6 +129,7 @@ type RestartReport struct { type RmOptions struct { All bool + Depend bool Force bool Ignore bool Latest bool @@ -136,11 +137,6 @@ type RmOptions struct { Volumes bool } -type RmReport struct { - Err error - Id string //nolint -} - type ContainerInspectReport struct { *define.InspectContainerData } diff --git a/pkg/domain/entities/engine.go b/pkg/domain/entities/engine.go index a8023f7cf..055af7ff9 100644 --- a/pkg/domain/entities/engine.go +++ b/pkg/domain/entities/engine.go @@ -40,6 +40,7 @@ type PodmanConfig struct { Identity string // ssh identity for connecting to server MaxWorks int // maximum number of parallel threads MemoryProfile string // Hidden: Should memory profile be taken + NoOut bool // Don't output to stdout RegistriesConf string // allows for specifying a custom registries.conf Remote bool // Connection to Podman API Service will use RESTful API RuntimePath string // --runtime flag will set Engine.RuntimePath diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index 383e42098..7ce4dd0f6 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -40,7 +40,7 @@ type ContainerEngine interface { ContainerRename(ctr context.Context, nameOrID string, options ContainerRenameOptions) error ContainerRestart(ctx context.Context, namesOrIds []string, options RestartOptions) ([]*RestartReport, error) ContainerRestore(ctx context.Context, namesOrIds []string, options RestoreOptions) ([]*RestoreReport, error) - ContainerRm(ctx context.Context, namesOrIds []string, options RmOptions) ([]*RmReport, error) + ContainerRm(ctx context.Context, namesOrIds []string, options RmOptions) ([]*reports.RmReport, error) ContainerRun(ctx context.Context, opts ContainerRunOptions) (*ContainerRunReport, error) ContainerRunlabel(ctx context.Context, label string, image string, args []string, opts ContainerRunlabelOptions) error ContainerStart(ctx context.Context, namesOrIds []string, options ContainerStartOptions) ([]*ContainerStartReport, error) diff --git a/pkg/domain/entities/reports/containers.go b/pkg/domain/entities/reports/containers.go new file mode 100644 index 000000000..54bcd092b --- /dev/null +++ b/pkg/domain/entities/reports/containers.go @@ -0,0 +1,28 @@ +package reports + +type RmReport struct { + Id string `json:"Id"` //nolint + Err error `json:"Err,omitempty"` +} + +func RmReportsIds(r []*RmReport) []string { + ids := make([]string, 0, len(r)) + for _, v := range r { + if v == nil || v.Id == "" { + continue + } + ids = append(ids, v.Id) + } + return ids +} + +func RmReportsErrs(r []*RmReport) []error { + errs := make([]error, 0, len(r)) + for _, v := range r { + if v == nil || v.Err == nil { + continue + } + errs = append(errs, v.Err) + } + return errs +} diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index bf4dcff62..a4522698e 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -301,27 +301,27 @@ func (ic *ContainerEngine) removeContainer(ctx context.Context, ctr *libpod.Cont return err } -func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, options entities.RmOptions) ([]*entities.RmReport, error) { - reports := []*entities.RmReport{} +func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, options entities.RmOptions) ([]*reports.RmReport, error) { + rmReports := []*reports.RmReport{} names := namesOrIds // Attempt to remove named containers directly from storage, if container is defined in libpod // this will fail and code will fall through to removing the container from libpod.` tmpNames := []string{} for _, ctr := range names { - report := entities.RmReport{Id: ctr} + report := reports.RmReport{Id: ctr} report.Err = ic.Libpod.RemoveStorageContainer(ctr, options.Force) switch errors.Cause(report.Err) { case nil: // remove container names that we successfully deleted - reports = append(reports, &report) + rmReports = append(rmReports, &report) case define.ErrNoSuchCtr, define.ErrCtrExists: // There is still a potential this is a libpod container tmpNames = append(tmpNames, ctr) default: if _, err := ic.Libpod.LookupContainer(ctr); errors.Cause(err) == define.ErrNoSuchCtr { // remove container failed, but not a libpod container - reports = append(reports, &report) + rmReports = append(rmReports, &report) continue } // attempt to remove as a libpod container @@ -340,23 +340,34 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, for _, ctr := range names { logrus.Debugf("Evicting container %q", ctr) - report := entities.RmReport{Id: ctr} + report := reports.RmReport{Id: ctr} _, err := ic.Libpod.EvictContainer(ctx, ctr, options.Volumes) if err != nil { if options.Ignore && errors.Cause(err) == define.ErrNoSuchCtr { logrus.Debugf("Ignoring error (--allow-missing): %v", err) - reports = append(reports, &report) + rmReports = append(rmReports, &report) continue } report.Err = err - reports = append(reports, &report) + rmReports = append(rmReports, &report) continue } - reports = append(reports, &report) + rmReports = append(rmReports, &report) } - return reports, nil + return rmReports, nil } + if !options.All && options.Depend { + // Add additional containers based on dependencies to container map + for _, ctr := range ctrs { + reports, err := ic.Libpod.RemoveDepend(ctx, ctr, options.Force, options.Volumes, options.Timeout) + if err != nil { + return rmReports, err + } + rmReports = append(rmReports, reports...) + } + return rmReports, nil + } errMap, err := parallelctr.ContainerOp(ctx, ctrs, func(c *libpod.Container) error { return ic.removeContainer(ctx, c, options) }) @@ -364,12 +375,12 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, return nil, err } for ctr, err := range errMap { - report := new(entities.RmReport) + report := new(reports.RmReport) report.Id = ctr.ID() report.Err = err - reports = append(reports, report) + rmReports = append(rmReports, report) } - return reports, nil + return rmReports, nil } func (ic *ContainerEngine) ContainerInspect(ctx context.Context, namesOrIds []string, options entities.InspectOptions) ([]*entities.ContainerInspectReport, []error, error) { diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 84c83ea8e..592e0f4e3 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -745,11 +745,17 @@ func putSignature(manifestBlob []byte, mech signature.SigningMechanism, sigStore // TransferRootless creates new podman processes using exec.Command and sudo, transferring images between the given source and destination users func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOptions, podman string, parentFlags []string) error { var cmdSave *exec.Cmd - saveCommand := parentFlags - saveCommand = append(saveCommand, []string{"save", "--output", source.File, source.Image}...) + saveCommand, loadCommand := parentFlags, parentFlags + saveCommand = append(saveCommand, []string{"save"}...) + loadCommand = append(loadCommand, []string{"load"}...) + if source.Quiet { + saveCommand = append(saveCommand, "-q") + loadCommand = append(loadCommand, "-q") + } + + saveCommand = append(saveCommand, []string{"--output", source.File, source.Image}...) - loadCommand := parentFlags - loadCommand = append(loadCommand, []string{"load", "--input", dest.File}...) + loadCommand = append(loadCommand, []string{"--input", dest.File}...) if source.User == "root" { cmdSave = exec.Command("sudo", podman) @@ -757,7 +763,7 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt cmdSave = exec.Command(podman) } cmdSave = utils.CreateSCPCommand(cmdSave, saveCommand) - logrus.Debug("Executing save command") + logrus.Debugf("Executing save command: %q", cmdSave) err := cmdSave.Run() if err != nil { return err @@ -770,20 +776,22 @@ func transferRootless(source entities.ImageScpOptions, dest entities.ImageScpOpt cmdLoad = exec.Command(podman) } cmdLoad = utils.CreateSCPCommand(cmdLoad, loadCommand) - logrus.Debug("Executing load command") - err = cmdLoad.Run() - if err != nil { - return err - } - return nil + logrus.Debugf("Executing load command: %q", cmdLoad) + return cmdLoad.Run() } -// TransferRootless creates new podman processes using exec.Command and su/machinectl, transferring images between the given source and destination users +// TransferRootful creates new podman processes using exec.Command and su/machinectl, transferring images between the given source and destination users func transferRootful(source entities.ImageScpOptions, dest entities.ImageScpOptions, podman string, parentFlags []string) error { basicCommand := []string{podman} basicCommand = append(basicCommand, parentFlags...) - saveCommand := append(basicCommand, []string{"save", "--output", source.File, source.Image}...) - loadCommand := append(basicCommand, []string{"load", "--input", dest.File}...) + saveCommand := append(basicCommand, "save") + loadCommand := append(basicCommand, "load") + if source.Quiet { + saveCommand = append(saveCommand, "-q") + loadCommand = append(loadCommand, "-q") + } + saveCommand = append(saveCommand, []string{"--output", source.File, source.Image}...) + loadCommand = append(loadCommand, []string{"--input", dest.File}...) save := []string{strings.Join(saveCommand, " ")} load := []string{strings.Join(loadCommand, " ")} @@ -846,18 +854,18 @@ func lookupUser(u string) (*user.User, error) { func execSu(execUser *user.User, command []string) error { cmd := exec.Command("su", "-l", execUser.Username, "--command") cmd = utils.CreateSCPCommand(cmd, command) - logrus.Debug("Executing command su") + logrus.Debugf("Executing via su: %q", cmd) return cmd.Run() } func execMachine(execUser *user.User, command []string, machinectl string) error { - var cmd *exec.Cmd + verb := machinectl + args := []string{"shell", "-q", execUser.Username + "@.host"} if execUser.Uid == "0" { - cmd = exec.Command("sudo", machinectl, "shell", "-q", execUser.Username+"@.host") - } else { - cmd = exec.Command(machinectl, "shell", "-q", execUser.Username+"@.host") + args = append([]string{verb}, args...) + verb = "sudo" } - cmd = utils.CreateSCPCommand(cmd, command) - logrus.Debug("Executing command machinectl") + cmd := utils.CreateSCPCommand(exec.Command(verb, args...), command) + logrus.Debugf("Executing via machinectl: %q", cmd) return cmd.Run() } diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index 2127f8749..4f72eab96 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -182,9 +182,9 @@ func (ic *ContainerEngine) ContainerRestart(ctx context.Context, namesOrIds []st return reports, nil } -func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, opts entities.RmOptions) ([]*entities.RmReport, error) { +func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, opts entities.RmOptions) ([]*reports.RmReport, error) { // TODO there is no endpoint for container eviction. Need to discuss - options := new(containers.RemoveOptions).WithForce(opts.Force).WithVolumes(opts.Volumes).WithIgnore(opts.Ignore) + options := new(containers.RemoveOptions).WithForce(opts.Force).WithVolumes(opts.Volumes).WithIgnore(opts.Ignore).WithDepend(opts.Depend) if opts.Timeout != nil { options = options.WithTimeout(*opts.Timeout) } @@ -193,25 +193,31 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, if err != nil { return nil, err } - reports := make([]*entities.RmReport, 0, len(ctrs)) + rmReports := make([]*reports.RmReport, 0, len(ctrs)) for _, c := range ctrs { - reports = append(reports, &entities.RmReport{ - Id: c.ID, - Err: containers.Remove(ic.ClientCtx, c.ID, options), - }) + report, err := containers.Remove(ic.ClientCtx, c.ID, options) + if err != nil { + return rmReports, err + } + rmReports = append(rmReports, report...) } - return reports, nil + return rmReports, nil } - reports := make([]*entities.RmReport, 0, len(namesOrIds)) + rmReports := make([]*reports.RmReport, 0, len(namesOrIds)) for _, name := range namesOrIds { - reports = append(reports, &entities.RmReport{ - Id: name, - Err: containers.Remove(ic.ClientCtx, name, options), - }) + report, err := containers.Remove(ic.ClientCtx, name, options) + if err != nil { + rmReports = append(rmReports, &reports.RmReport{ + Id: name, + Err: err, + }) + continue + } + rmReports = append(rmReports, report...) } - return reports, nil + return rmReports, nil } func (ic *ContainerEngine) ContainerPrune(ctx context.Context, opts entities.ContainerPruneOptions) ([]*reports.PruneReport, error) { @@ -552,6 +558,27 @@ func startAndAttach(ic *ContainerEngine, name string, detachKeys *string, input, return <-attachErr } +func logIfRmError(id string, err error, reports []*reports.RmReport) { + logError := func(id string, err error) { + if errorhandling.Contains(err, define.ErrNoSuchCtr) || + errorhandling.Contains(err, define.ErrCtrRemoved) || + errorhandling.Contains(err, types.ErrLayerUnknown) { + logrus.Debugf("Container %s does not exist: %v", id, err) + } else { + logrus.Errorf("Removing container %s: %v", id, err) + } + } + if err != nil { + logError(id, err) + } else { + for _, report := range reports { + if report.Err != nil { + logError(report.Id, report.Err) + } + } + } +} + func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []string, options entities.ContainerStartOptions) ([]*entities.ContainerStartReport, error) { reports := []*entities.ContainerStartReport{} var exitCode = define.ExecErrorCodeGeneric @@ -590,14 +617,8 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri } removeOptions := new(containers.RemoveOptions).WithVolumes(true).WithForce(false) removeContainer := func(id string) { - if err := containers.Remove(ic.ClientCtx, id, removeOptions); err != nil { - if errorhandling.Contains(err, define.ErrNoSuchCtr) || - errorhandling.Contains(err, define.ErrCtrRemoved) { - logrus.Debugf("Container %s does not exist: %v", id, err) - } else { - logrus.Errorf("Removing container %s: %v", id, err) - } - } + reports, err := containers.Remove(ic.ClientCtx, id, removeOptions) + logIfRmError(id, err, reports) } // There can only be one container if attach was used @@ -674,15 +695,8 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri if err != nil { if ctr.AutoRemove { rmOptions := new(containers.RemoveOptions).WithForce(false).WithVolumes(true) - if err := containers.Remove(ic.ClientCtx, ctr.ID, rmOptions); err != nil { - if errorhandling.Contains(err, define.ErrNoSuchCtr) || - errorhandling.Contains(err, define.ErrCtrRemoved) || - errorhandling.Contains(err, types.ErrLayerUnknown) { - logrus.Debugf("Container %s does not exist: %v", ctr.ID, err) - } else { - logrus.Errorf("Removing container %s: %v", ctr.ID, err) - } - } + reports, err := containers.Remove(ic.ClientCtx, ctr.ID, rmOptions) + logIfRmError(ctr.ID, err, reports) } report.Err = errors.Wrapf(err, "unable to start container %q", name) report.ExitCode = define.ExitCode(err) @@ -741,7 +755,8 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta report.ExitCode = define.ExitCode(err) if opts.Rm { - if rmErr := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)); rmErr != nil { + reports, rmErr := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)) + if rmErr != nil || reports[0].Err != nil { logrus.Debugf("unable to remove container %s after failing to start and attach to it", con.ID) } } @@ -759,15 +774,8 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta } if !shouldRestart { - if err := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)); err != nil { - if errorhandling.Contains(err, define.ErrNoSuchCtr) || - errorhandling.Contains(err, define.ErrCtrRemoved) || - errorhandling.Contains(err, types.ErrLayerUnknown) { - logrus.Debugf("Container %s does not exist: %v", con.ID, err) - } else { - logrus.Errorf("Removing container %s: %v", con.ID, err) - } - } + reports, err := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)) + logIfRmError(con.ID, err, reports) } }() } diff --git a/pkg/machine/config.go b/pkg/machine/config.go index 252ad9768..97237f5e5 100644 --- a/pkg/machine/config.go +++ b/pkg/machine/config.go @@ -75,15 +75,18 @@ type Download struct { type ListOptions struct{} type ListResponse struct { - Name string - CreatedAt time.Time - LastUp time.Time - Running bool - Stream string - VMType string - CPUs uint64 - Memory uint64 - DiskSize uint64 + Name string + CreatedAt time.Time + LastUp time.Time + Running bool + Stream string + VMType string + CPUs uint64 + Memory uint64 + DiskSize uint64 + Port int + RemoteUsername string + IdentityPath string } type SSHOptions struct { diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 07f40984c..560037542 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -790,6 +790,9 @@ func GetVMInfos() ([]*machine.ListResponse, error) { listEntry.CPUs = vm.CPUs listEntry.Memory = vm.Memory * units.MiB listEntry.DiskSize = vm.DiskSize * units.GiB + listEntry.Port = vm.Port + listEntry.RemoteUsername = vm.RemoteUsername + listEntry.IdentityPath = vm.IdentityPath fi, err := os.Stat(fullPath) if err != nil { return err diff --git a/pkg/specgen/generate/ports.go b/pkg/specgen/generate/ports.go index b60cc1e98..34b43a62e 100644 --- a/pkg/specgen/generate/ports.go +++ b/pkg/specgen/generate/ports.go @@ -206,7 +206,7 @@ func ParsePortMapping(portMappings []types.PortMapping, exposePorts map[uint16][ } // we do no longer need the original port mappings - // set it to 0 length so we can resuse it to populate + // set it to 0 length so we can reuse it to populate // the slice again while keeping the underlying capacity portMappings = portMappings[:0] diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 33e8422fd..fdaa714da 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -95,7 +95,7 @@ type PodNetworkConfig struct { // Map of networks names ot ids the container should join to. // You can request additional settings for each network, you can // set network aliases, static ips, static mac address and the - // network interface name for this container on the specifc network. + // network interface name for this container on the specific network. // If the map is empty and the bridge network mode is set the container // will be joined to the default network. Networks map[string]types.PerNetworkOptions diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index 5989456c9..6c1011a78 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -426,7 +426,7 @@ type ContainerNetworkConfig struct { // Map of networks names ot ids the container should join to. // You can request additional settings for each network, you can // set network aliases, static ips, static mac address and the - // network interface name for this container on the specifc network. + // network interface name for this container on the specific network. // If the map is empty and the bridge network mode is set the container // will be joined to the default network. Networks map[string]nettypes.PerNetworkOptions diff --git a/test/apiv2/12-imagesMore.at b/test/apiv2/12-imagesMore.at index 3a5d5c096..96eba1db5 100644 --- a/test/apiv2/12-imagesMore.at +++ b/test/apiv2/12-imagesMore.at @@ -53,7 +53,7 @@ t GET libpod/images/$IMAGE/json 200 \ .RepoTags[-1]=$IMAGE # Remove the registry container -t DELETE libpod/containers/registry?force=true 204 +t DELETE libpod/containers/registry?force=true 200 # Remove images t DELETE libpod/images/$IMAGE 200 \ diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 554a905d4..936597f72 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -85,7 +85,7 @@ else fi fi -t DELETE libpod/containers/$cid 204 +t DELETE libpod/containers/$cid 200 .[0].Id=$cid # Issue #6799: it should be possible to start a container, even w/o args. t POST libpod/containers/create?name=test_noargs Image=${IMAGE} 201 \ @@ -100,7 +100,7 @@ t GET libpod/containers/${cid}/json 200 \ .State.Status~\\\(exited\\\|stopped\\\) \ .State.Running=false \ .State.ExitCode=0 -t DELETE libpod/containers/$cid 204 +t DELETE libpod/containers/$cid 200 .[0].Id=$cid CNAME=myfoo podman run -d --name $CNAME $IMAGE top @@ -190,8 +190,8 @@ t GET containers/myctr/json 200 \ t DELETE images/localhost/newrepo:latest?force=true 200 t DELETE images/localhost/newrepo:v1?force=true 200 t DELETE images/localhost/newrepo:v2?force=true 200 -t DELETE libpod/containers/$cid?force=true 204 -t DELETE libpod/containers/myctr 204 +t DELETE libpod/containers/$cid?force=true 200 .[0].Id=$cid +t DELETE libpod/containers/myctr 200 t DELETE libpod/containers/bogus 404 diff --git a/test/apiv2/22-stop.at b/test/apiv2/22-stop.at index 91bc9937d..bde534b72 100644 --- a/test/apiv2/22-stop.at +++ b/test/apiv2/22-stop.at @@ -11,7 +11,7 @@ podman run -dt --name mytop $IMAGE top &>/dev/null t GET libpod/containers/mytop/json 200 .State.Status=running t POST libpod/containers/mytop/stop 204 t GET libpod/containers/mytop/json 200 .State.Status~\\\(exited\\\|stopped\\\) -t DELETE libpod/containers/mytop 204 +t DELETE libpod/containers/mytop 200 # stop, by ID # Remember that podman() hides all output; we need to get our CID via inspect @@ -21,4 +21,4 @@ t GET libpod/containers/mytop/json 200 .State.Status=running cid=$(jq -r .Id <<<"$output") t POST libpod/containers/$cid/stop 204 t GET libpod/containers/mytop/json 200 .State.Status~\\\(exited\\\|stopped\\\) -t DELETE libpod/containers/mytop 204 +t DELETE libpod/containers/mytop 200 diff --git a/test/apiv2/25-containersMore.at b/test/apiv2/25-containersMore.at index 0a049d869..c9fda8c6f 100644 --- a/test/apiv2/25-containersMore.at +++ b/test/apiv2/25-containersMore.at @@ -51,7 +51,7 @@ like "$output" ".*merged" "Check container mount" # Unmount the container t POST libpod/containers/foo/unmount 204 -t DELETE libpod/containers/foo?force=true 204 +t DELETE libpod/containers/foo?force=true 200 podman run $IMAGE true @@ -79,7 +79,7 @@ like "$output" ".*metadata:.*" "Check generated kube yaml(service=true) - metada like "$output" ".*spec:.*" "Check generated kube yaml(service=true) - spec" like "$output" ".*kind:\\sService.*" "Check generated kube yaml(service=true) - kind: Service" -t DELETE libpod/containers/$cid 204 +t DELETE libpod/containers/$cid 200 .[0].Id=$cid # Create 3 stopped containers to test containers prune podman run $IMAGE true diff --git a/test/apiv2/python/rest_api/test_v2_0_0_container.py b/test/apiv2/python/rest_api/test_v2_0_0_container.py index 101044bbb..1b4597cf8 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_container.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_container.py @@ -99,7 +99,7 @@ class ContainerTestCase(APITestCase): def test_delete(self): r = requests.delete(self.uri(self.resolve_container("/containers/{}?force=true"))) - self.assertEqual(r.status_code, 204, r.text) + self.assertEqual(r.status_code, 200, r.text) def test_stop(self): r = requests.post(self.uri(self.resolve_container("/containers/{}/start"))) diff --git a/test/buildah-bud/apply-podman-deltas b/test/buildah-bud/apply-podman-deltas index 44a33b0b8..e42c8aa52 100755 --- a/test/buildah-bud/apply-podman-deltas +++ b/test/buildah-bud/apply-podman-deltas @@ -191,6 +191,16 @@ skip_if_remote "--stdin option will not be implemented in podman-remote" \ # BEGIN tests which are skipped due to actual podman-remote bugs. ############################################################################### +# BEGIN emergency handling of github git-protocol shutdown +# +# Please remove this as soon as we vendor buildah with #3701 + +skip "emergency workaround until buildah #3701 gets vendored in" \ + "bud-git-context" \ + "bud using gitrepo and branch" + +# END emergency handling of github git-protocol shutdown +############################################################################### # Done. exit $RC diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats index 23489c1b5..9b0a71285 100644 --- a/test/system/001-basic.bats +++ b/test/system/001-basic.bats @@ -41,7 +41,7 @@ function setup() { # This one must fail run_podman 125 --context=swarm version is "$output" \ - "Error: Podman does not support swarm, the only --context value allowed is \"default\"" \ + "Error: podman does not support swarm, the only --context value allowed is \"default\"" \ "--context=default or fail" } diff --git a/test/system/030-run.bats b/test/system/030-run.bats index feca5370b..afcda3d3c 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -509,6 +509,21 @@ json-file | f rm -f $new_runtime } +@test "podman --noout run should print output" { + run_podman --noout run -d --name test $IMAGE echo hi + is "$output" "" "output should be empty" + run_podman wait test + run_podman --noout rm test + is "$output" "" "output should be empty" +} + +@test "podman --noout create should print output" { + run_podman --noout create --name test $IMAGE echo hi + is "$output" "" "output should be empty" + run_podman --noout rm test + is "$output" "" "output should be empty" +} + # Regression test for issue #8082 @test "podman run : look up correct image name" { # Create a 2nd tag for the local image. Force to lower case, and apply it. diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats index e049da518..7dd8f98e8 100644 --- a/test/system/050-stop.bats +++ b/test/system/050-stop.bats @@ -173,4 +173,9 @@ load helpers is "$output" ".*StopSignal SIGTERM failed to stop container stopme in 1 seconds, resorting to SIGKILL" "stopping container should print warning" } +@test "podman stop --noout" { + run_podman run --rm --name stopme -d $IMAGE top + run_podman --noout stop -t 0 stopme + is "$output" "" "output should be empty" +} # vim: filetype=sh diff --git a/test/system/055-rm.bats b/test/system/055-rm.bats index 7fe81c084..69663fafa 100644 --- a/test/system/055-rm.bats +++ b/test/system/055-rm.bats @@ -58,6 +58,18 @@ load helpers run_podman rm -af } +@test "podman rm --depend" { + run_podman create $IMAGE + dependCid=$output + run_podman create --net=container:$dependCid $IMAGE + cid=$output + run_podman 125 rm $dependCid + is "$output" "Error: container $dependCid has dependent containers which must be removed before it:.*" "Fail to remove because of dependencies" + run_podman rm --depend $dependCid + is "$output" ".*$cid" "Container should have been removed" + is "$output" ".*$dependCid" "Depend container should have been removed" +} + # I'm sorry! This test takes 13 seconds. There's not much I can do about it, # please know that I think it's justified: podman 1.5.0 had a strange bug # in with exit status was not preserved on some code paths with 'rm -f' diff --git a/test/system/080-pause.bats b/test/system/080-pause.bats index 857c8bbf4..57f390a74 100644 --- a/test/system/080-pause.bats +++ b/test/system/080-pause.bats @@ -21,7 +21,8 @@ load helpers # time to write a new post-restart time value. Pause by CID, unpause # by name, just to exercise code paths. While paused, check 'ps' # and 'inspect', then check again after restarting. - run_podman pause $cid + run_podman --noout pause $cid + is "$output" "" "output should be empty" run_podman inspect --format '{{.State.Status}}' $cid is "$output" "paused" "podman inspect .State.Status" sleep 3 diff --git a/test/system/170-run-userns.bats b/test/system/170-run-userns.bats index a5be591ef..c020a73ab 100644 --- a/test/system/170-run-userns.bats +++ b/test/system/170-run-userns.bats @@ -78,3 +78,19 @@ EOF # Then check that the main user is not mapped into the user namespace CONTAINERS_CONF=$PODMAN_TMPDIR/userns_auto.conf run_podman 0 run --rm $IMAGE awk '{if($2 == "0"){exit 1}}' /proc/self/uid_map /proc/self/gid_map } + +@test "podman userns=auto and secrets" { + ns_user="containers" + if is_rootless; then + ns_user=$(id -un) + fi + egrep -q "${ns_user}:" /etc/subuid || skip "no IDs allocated for user ${ns_user}" + test_name="test_$(random_string 12)" + secret_file=$PODMAN_TMPDIR/secret$(random_string 12) + secret_content=$(random_string) + echo ${secret_content} > ${secret_file} + run_podman secret create ${test_name} ${secret_file} + run_podman run --rm --secret=${test_name} --userns=auto:size=1000 $IMAGE cat /run/secrets/${test_name} + is ${output} ${secret_content} "Secrets should work with user namespace" + run_podman secret rm ${test_name} +} diff --git a/test/system/200-pod.bats b/test/system/200-pod.bats index 6abdf9779..4a3337e57 100644 --- a/test/system/200-pod.bats +++ b/test/system/200-pod.bats @@ -57,7 +57,8 @@ function teardown() { fi # Clean up - run_podman pod rm -f -t 0 $podid + run_podman --noout pod rm -f -t 0 $podid + is "$output" "" "output should be empty" } @@ -330,7 +331,8 @@ EOF # Note that the internal pause image is built even when --infra-image is # set to the K8s one. - run_podman pod create --name $pod_name --infra-name "$infra_name" --infra-image "k8s.gcr.io/pause:3.5" + run_podman --noout pod create --name $pod_name --infra-name "$infra_name" --infra-image "k8s.gcr.io/pause:3.5" + is "$output" "" "output should be empty" run_podman '?' pod create --infra-name "$infra_name" if [ $status -eq 0 ]; then die "Podman should fail when user try to create two pods with the same infra-name value" diff --git a/test/system/250-systemd.bats b/test/system/250-systemd.bats index c49727cc9..c47679904 100644 --- a/test/system/250-systemd.bats +++ b/test/system/250-systemd.bats @@ -276,4 +276,9 @@ LISTEN_FDNAMES=listen_fdnames" | sort) is "$output" ".*--template cannot be set" "Error message should be '--template requires --new'" } +@test "podman --cgroup=cgroupfs doesn't show systemd warning" { + DBUS_SESSION_BUS_ADDRESS= run_podman --log-level warning --cgroup-manager=cgroupfs info -f '' + is "$output" "" "output should be empty" +} + # vim: filetype=sh diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index 2b5ad44dc..5a721c965 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -332,7 +332,8 @@ load helpers is_rootless || skip "only meaningful for rootless" local mynetname=testnet-$(random_string 10) - run_podman network create $mynetname + run_podman --noout network create $mynetname + is "$output" "" "output should be empty" # Test that rootless cni adds /usr/sbin to $PATH # iptables is located under /usr/sbin and is needed for the CNI plugins. @@ -340,7 +341,8 @@ load helpers PATH=/usr/local/bin:/usr/bin run_podman run --rm --network $mynetname $IMAGE ip addr is "$output" ".*eth0.*" "Interface eth0 not found in ip addr output" - run_podman network rm -t 0 -f $mynetname + run_podman --noout network rm -t 0 -f $mynetname + is "$output" "" "output should be empty" } @test "podman ipv6 in /etc/resolv.conf" { diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md index 64410cf75..cc13f8667 100644 --- a/vendor/github.com/BurntSushi/toml/README.md +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -1,10 +1,6 @@ -## TOML parser and encoder for Go with reflection - TOML stands for Tom's Obvious, Minimal Language. This Go package provides a reflection interface similar to Go's standard library `json` and `xml` -packages. This package also supports the `encoding.TextUnmarshaler` and -`encoding.TextMarshaler` interfaces so that you can define custom data -representations. (There is an example of this below.) +packages. Compatible with TOML version [v1.0.0](https://toml.io/en/v1.0.0). @@ -16,26 +12,25 @@ v0.4.0`). This library requires Go 1.13 or newer; install it with: - $ go get github.com/BurntSushi/toml + % go get github.com/BurntSushi/toml@latest It also comes with a TOML validator CLI tool: - $ go get github.com/BurntSushi/toml/cmd/tomlv - $ tomlv some-toml-file.toml + % go install github.com/BurntSushi/toml/cmd/tomlv@latest + % tomlv some-toml-file.toml ### Testing +This package passes all tests in [toml-test] for both the decoder and the +encoder. -This package passes all tests in -[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder -and the encoder. +[toml-test]: https://github.com/BurntSushi/toml-test ### Examples +This package works similar to how the Go standard library handles XML and JSON. +Namely, data is loaded into Go values via reflection. -This package works similarly to how the Go standard library handles XML and -JSON. Namely, data is loaded into Go values via reflection. - -For the simplest example, consider some TOML file as just a list of keys -and values: +For the simplest example, consider some TOML file as just a list of keys and +values: ```toml Age = 25 @@ -61,9 +56,8 @@ And then decoded with: ```go var conf Config -if _, err := toml.Decode(tomlData, &conf); err != nil { - // handle error -} +err := toml.Decode(tomlData, &conf) +// handle error ``` You can also use struct tags if your struct field name doesn't map to a TOML @@ -75,15 +69,14 @@ some_key_NAME = "wat" ```go type TOML struct { - ObscureKey string `toml:"some_key_NAME"` + ObscureKey string `toml:"some_key_NAME"` } ``` Beware that like other most other decoders **only exported fields** are considered when encoding and decoding; private fields are silently ignored. -### Using the `encoding.TextUnmarshaler` interface - +### Using the `Marshaler` and `encoding.TextUnmarshaler` interfaces Here's an example that automatically parses duration strings into `time.Duration` values: @@ -136,7 +129,6 @@ To target TOML specifically you can implement `UnmarshalTOML` TOML interface in a similar way. ### More complex usage - Here's an example of how to load the example from the official spec page: ```toml @@ -216,5 +208,4 @@ type clients struct { Note that a case insensitive match will be tried if an exact match can't be found. -A working example of the above can be found in `_examples/example.{go,toml}`. - +A working example of the above can be found in `_example/example.{go,toml}`. diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index d3d3b8397..e24f0c5d5 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -9,7 +9,6 @@ import ( "os" "reflect" "strings" - "time" ) // Unmarshaler is the interface implemented by objects that can unmarshal a @@ -40,6 +39,13 @@ type Primitive struct { context Key } +// The significand precision for float32 and float64 is 24 and 53 bits; this is +// the range a natural number can be stored in a float without loss of data. +const ( + maxSafeFloat32Int = 16777215 // 2^24-1 + maxSafeFloat64Int = 9007199254740991 // 2^53-1 +) + // PrimitiveDecode is just like the other `Decode*` functions, except it // decodes a TOML value that has already been parsed. Valid primitive values // can *only* be obtained from values filled by the decoder functions, @@ -100,18 +106,38 @@ func NewDecoder(r io.Reader) *Decoder { return &Decoder{r: r} } +var ( + unmarshalToml = reflect.TypeOf((*Unmarshaler)(nil)).Elem() + unmarshalText = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() +) + // Decode TOML data in to the pointer `v`. func (dec *Decoder) Decode(v interface{}) (MetaData, error) { rv := reflect.ValueOf(v) if rv.Kind() != reflect.Ptr { - return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + s := "%q" + if reflect.TypeOf(v) == nil { + s = "%v" + } + + return MetaData{}, e("cannot decode to non-pointer "+s, reflect.TypeOf(v)) } if rv.IsNil() { - return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + return MetaData{}, e("cannot decode to nil value of %q", reflect.TypeOf(v)) + } + + // Check if this is a supported type: struct, map, interface{}, or something + // that implements UnmarshalTOML or UnmarshalText. + rv = indirect(rv) + rt := rv.Type() + if rv.Kind() != reflect.Struct && rv.Kind() != reflect.Map && + !(rv.Kind() == reflect.Interface && rv.NumMethod() == 0) && + !rt.Implements(unmarshalToml) && !rt.Implements(unmarshalText) { + return MetaData{}, e("cannot decode to type %s", rt) } - // TODO: have parser should read from io.Reader? Or at the very least, make - // it read from []byte rather than string + // TODO: parser should read from io.Reader? Or at the very least, make it + // read from []byte rather than string data, err := ioutil.ReadAll(dec.r) if err != nil { return MetaData{}, err @@ -121,11 +147,15 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) { if err != nil { return MetaData{}, err } + md := MetaData{ - p.mapping, p.types, p.ordered, - make(map[string]bool, len(p.ordered)), nil, + mapping: p.mapping, + types: p.types, + keys: p.ordered, + decoded: make(map[string]struct{}, len(p.ordered)), + context: nil, } - return md, md.unify(p.mapping, indirect(rv)) + return md, md.unify(p.mapping, rv) } // Decode the TOML data in to the pointer v. @@ -218,9 +248,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { return e("unsupported type %s", rv.Type()) } return md.unifyAnything(data, rv) - case reflect.Float32: - fallthrough - case reflect.Float64: + case reflect.Float32, reflect.Float64: return md.unifyFloat64(data, rv) } return e("unsupported type %s", rv.Kind()) @@ -254,17 +282,17 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { for _, i := range f.index { subv = indirect(subv.Field(i)) } + if isUnifiable(subv) { - md.decoded[md.context.add(key).String()] = true + md.decoded[md.context.add(key).String()] = struct{}{} md.context = append(md.context, key) - if err := md.unify(datum, subv); err != nil { + err := md.unify(datum, subv) + if err != nil { return err } md.context = md.context[0 : len(md.context)-1] } else if f.name != "" { - // Bad user! No soup for you! - return e("cannot write unexported field %s.%s", - rv.Type().String(), f.name) + return e("cannot write unexported field %s.%s", rv.Type().String(), f.name) } } } @@ -283,22 +311,22 @@ func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { if tmap == nil { return nil } - return badtype("map", mapping) + return md.badtype("map", mapping) } if rv.IsNil() { rv.Set(reflect.MakeMap(rv.Type())) } for k, v := range tmap { - md.decoded[md.context.add(k).String()] = true + md.decoded[md.context.add(k).String()] = struct{}{} md.context = append(md.context, k) - rvkey := indirect(reflect.New(rv.Type().Key())) rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) if err := md.unify(v, rvval); err != nil { return err } md.context = md.context[0 : len(md.context)-1] + rvkey := indirect(reflect.New(rv.Type().Key())) rvkey.SetString(k) rv.SetMapIndex(rvkey, rvval) } @@ -311,7 +339,7 @@ func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { if !datav.IsValid() { return nil } - return badtype("slice", data) + return md.badtype("slice", data) } if l := datav.Len(); l != rv.Len() { return e("expected array length %d; got TOML array of length %d", rv.Len(), l) @@ -325,7 +353,7 @@ func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { if !datav.IsValid() { return nil } - return badtype("slice", data) + return md.badtype("slice", data) } n := datav.Len() if rv.IsNil() || rv.Cap() < n { @@ -346,26 +374,21 @@ func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { return nil } -func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { - if _, ok := data.(time.Time); ok { - rv.Set(reflect.ValueOf(data)) - return nil - } - return badtype("time.Time", data) -} - func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { if s, ok := data.(string); ok { rv.SetString(s) return nil } - return badtype("string", data) + return md.badtype("string", data) } func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { if num, ok := data.(float64); ok { switch rv.Kind() { case reflect.Float32: + if num < -math.MaxFloat32 || num > math.MaxFloat32 { + return e("value %f is out of range for float32", num) + } fallthrough case reflect.Float64: rv.SetFloat(num) @@ -374,7 +397,26 @@ func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { } return nil } - return badtype("float", data) + + if num, ok := data.(int64); ok { + switch rv.Kind() { + case reflect.Float32: + if num < -maxSafeFloat32Int || num > maxSafeFloat32Int { + return e("value %d is out of range for float32", num) + } + fallthrough + case reflect.Float64: + if num < -maxSafeFloat64Int || num > maxSafeFloat64Int { + return e("value %d is out of range for float64", num) + } + rv.SetFloat(float64(num)) + default: + panic("bug") + } + return nil + } + + return md.badtype("float", data) } func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { @@ -421,7 +463,7 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { } return nil } - return badtype("integer", data) + return md.badtype("integer", data) } func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { @@ -429,7 +471,7 @@ func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { rv.SetBool(b) return nil } - return badtype("boolean", data) + return md.badtype("boolean", data) } func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { @@ -440,6 +482,12 @@ func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) error { var s string switch sdata := data.(type) { + case Marshaler: + text, err := sdata.MarshalTOML() + if err != nil { + return err + } + s = string(text) case TextMarshaler: text, err := sdata.MarshalText() if err != nil { @@ -457,7 +505,7 @@ func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) erro case float64: s = fmt.Sprintf("%f", sdata) default: - return badtype("primitive (string-like)", data) + return md.badtype("primitive (string-like)", data) } if err := v.UnmarshalText([]byte(s)); err != nil { return err @@ -465,17 +513,22 @@ func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) erro return nil } +func (md *MetaData) badtype(dst string, data interface{}) error { + return e("incompatible types: TOML key %q has type %T; destination has type %s", md.context, data, dst) +} + // rvalue returns a reflect.Value of `v`. All pointers are resolved. func rvalue(v interface{}) reflect.Value { return indirect(reflect.ValueOf(v)) } // indirect returns the value pointed to by a pointer. -// Pointers are followed until the value is not a pointer. -// New values are allocated for each nil pointer. // -// An exception to this rule is if the value satisfies an interface of -// interest to us (like encoding.TextUnmarshaler). +// Pointers are followed until the value is not a pointer. New values are +// allocated for each nil pointer. +// +// An exception to this rule is if the value satisfies an interface of interest +// to us (like encoding.TextUnmarshaler). func indirect(v reflect.Value) reflect.Value { if v.Kind() != reflect.Ptr { if v.CanSet() { @@ -505,7 +558,3 @@ func isUnifiable(rv reflect.Value) bool { func e(format string, args ...interface{}) error { return fmt.Errorf("toml: "+format, args...) } - -func badtype(expected string, data interface{}) error { - return e("cannot load TOML value of type %T into a Go %s", data, expected) -} diff --git a/vendor/github.com/BurntSushi/toml/decode_go116.go b/vendor/github.com/BurntSushi/toml/decode_go116.go index 38aa75fdc..eddfb641b 100644 --- a/vendor/github.com/BurntSushi/toml/decode_go116.go +++ b/vendor/github.com/BurntSushi/toml/decode_go116.go @@ -1,3 +1,4 @@ +//go:build go1.16 // +build go1.16 package toml diff --git a/vendor/github.com/BurntSushi/toml/deprecated.go b/vendor/github.com/BurntSushi/toml/deprecated.go index db89eac1d..c6af3f239 100644 --- a/vendor/github.com/BurntSushi/toml/deprecated.go +++ b/vendor/github.com/BurntSushi/toml/deprecated.go @@ -5,29 +5,17 @@ import ( "io" ) -// DEPRECATED! -// -// Use the identical encoding.TextMarshaler instead. It is defined here to -// support Go 1.1 and older. +// Deprecated: use encoding.TextMarshaler type TextMarshaler encoding.TextMarshaler -// DEPRECATED! -// -// Use the identical encoding.TextUnmarshaler instead. It is defined here to -// support Go 1.1 and older. +// Deprecated: use encoding.TextUnmarshaler type TextUnmarshaler encoding.TextUnmarshaler -// DEPRECATED! -// -// Use MetaData.PrimitiveDecode instead. +// Deprecated: use MetaData.PrimitiveDecode. func PrimitiveDecode(primValue Primitive, v interface{}) error { - md := MetaData{decoded: make(map[string]bool)} + md := MetaData{decoded: make(map[string]struct{})} return md.unify(primValue.undecoded, rvalue(v)) } -// DEPRECATED! -// -// Use NewDecoder(reader).Decode(&v) instead. -func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { - return NewDecoder(r).Decode(v) -} +// Deprecated: use NewDecoder(reader).Decode(&value). +func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { return NewDecoder(r).Decode(v) } diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index 10d88ac63..dee4e6d31 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -21,12 +21,11 @@ type tomlEncodeError struct{ error } var ( errArrayNilElement = errors.New("toml: cannot encode array with nil element") errNonString = errors.New("toml: cannot encode a map with non-string key type") - errAnonNonStruct = errors.New("toml: cannot encode an anonymous field that is not a struct") errNoKey = errors.New("toml: top-level values must be Go maps or structs") errAnything = errors.New("") // used in testing ) -var quotedReplacer = strings.NewReplacer( +var dblQuotedReplacer = strings.NewReplacer( "\"", "\\\"", "\\", "\\\\", "\x00", `\u0000`, @@ -64,13 +63,22 @@ var quotedReplacer = strings.NewReplacer( "\x7f", `\u007f`, ) +// Marshaler is the interface implemented by types that can marshal themselves +// into valid TOML. +type Marshaler interface { + MarshalTOML() ([]byte, error) +} + // Encoder encodes a Go to a TOML document. // // The mapping between Go values and TOML values should be precisely the same as -// for the Decode* functions. Similarly, the TextMarshaler interface is -// supported by encoding the resulting bytes as strings. If you want to write -// arbitrary binary data then you will need to use something like base64 since -// TOML does not have any binary types. +// for the Decode* functions. +// +// The toml.Marshaler and encoder.TextMarshaler interfaces are supported to +// encoding the value as custom TOML. +// +// If you want to write arbitrary binary data then you will need to use +// something like base64 since TOML does not have any binary types. // // When encoding TOML hashes (Go maps or structs), keys without any sub-hashes // are encoded first. @@ -83,16 +91,14 @@ var quotedReplacer = strings.NewReplacer( // structs. (e.g. [][]map[string]string is not allowed but []map[string]string // is okay, as is []map[string][]string). // -// NOTE: Only exported keys are encoded due to the use of reflection. Unexported +// NOTE: only exported keys are encoded due to the use of reflection. Unexported // keys are silently discarded. type Encoder struct { - // The string to use for a single indentation level. The default is two - // spaces. + // String to use for a single indentation level; default is two spaces. Indent string - // hasWritten is whether we have written any output to w yet. - hasWritten bool w *bufio.Writer + hasWritten bool // written any output to w yet? } // NewEncoder create a new Encoder. @@ -130,12 +136,13 @@ func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { } func (enc *Encoder) encode(key Key, rv reflect.Value) { - // Special case. Time needs to be in ISO8601 format. - // Special case. If we can marshal the type to text, then we used that. - // Basically, this prevents the encoder for handling these types as - // generic structs (or whatever the underlying type of a TextMarshaler is). + // Special case: time needs to be in ISO8601 format. + // + // Special case: if we can marshal the type to text, then we used that. This + // prevents the encoder for handling these types as generic structs (or + // whatever the underlying type of a TextMarshaler is). switch t := rv.Interface().(type) { - case time.Time, encoding.TextMarshaler: + case time.Time, encoding.TextMarshaler, Marshaler: enc.writeKeyValue(key, rv, false) return // TODO: #76 would make this superfluous after implemented. @@ -200,13 +207,19 @@ func (enc *Encoder) eElement(rv reflect.Value) { enc.wf(v.In(time.UTC).Format(format)) } return + case Marshaler: + s, err := v.MarshalTOML() + if err != nil { + encPanic(err) + } + enc.writeQuoted(string(s)) + return case encoding.TextMarshaler: - // Use text marshaler if it's available for this value. - if s, err := v.MarshalText(); err != nil { + s, err := v.MarshalText() + if err != nil { encPanic(err) - } else { - enc.writeQuoted(string(s)) } + enc.writeQuoted(string(s)) return } @@ -260,7 +273,7 @@ func floatAddDecimal(fstr string) string { } func (enc *Encoder) writeQuoted(s string) { - enc.wf("\"%s\"", quotedReplacer.Replace(s)) + enc.wf("\"%s\"", dblQuotedReplacer.Replace(s)) } func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { @@ -286,7 +299,7 @@ func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { continue } enc.newline() - enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) + enc.wf("%s[[%s]]", enc.indentStr(key), key) enc.newline() enc.eMapOrStruct(key, trv, false) } @@ -299,7 +312,7 @@ func (enc *Encoder) eTable(key Key, rv reflect.Value) { enc.newline() } if len(key) > 0 { - enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) + enc.wf("%s[%s]", enc.indentStr(key), key) enc.newline() } enc.eMapOrStruct(key, rv, false) @@ -328,7 +341,7 @@ func (enc *Encoder) eMap(key Key, rv reflect.Value, inline bool) { var mapKeysDirect, mapKeysSub []string for _, mapKey := range rv.MapKeys() { k := mapKey.String() - if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { + if typeIsTable(tomlTypeOfGo(rv.MapIndex(mapKey))) { mapKeysSub = append(mapKeysSub, k) } else { mapKeysDirect = append(mapKeysDirect, k) @@ -364,6 +377,8 @@ func (enc *Encoder) eMap(key Key, rv reflect.Value, inline bool) { } } +const is32Bit = (32 << (^uint(0) >> 63)) == 32 + func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) { // Write keys for fields directly under this key first, because if we write // a field that creates a new table then all keys under it will be in that @@ -408,10 +423,20 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) { } } - if typeIsHash(tomlTypeOfGo(frv)) { + if typeIsTable(tomlTypeOfGo(frv)) { fieldsSub = append(fieldsSub, append(start, f.Index...)) } else { - fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + // Copy so it works correct on 32bit archs; not clear why this + // is needed. See #314, and https://www.reddit.com/r/golang/comments/pnx8v4 + // This also works fine on 64bit, but 32bit archs are somewhat + // rare and this is a wee bit faster. + if is32Bit { + copyStart := make([]int, len(start)) + copy(copyStart, start) + fieldsDirect = append(fieldsDirect, append(copyStart, f.Index...)) + } else { + fieldsDirect = append(fieldsDirect, append(start, f.Index...)) + } } } } @@ -462,13 +487,13 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value, inline bool) { } } -// tomlTypeName returns the TOML type name of the Go value's type. It is -// used to determine whether the types of array elements are mixed (which is -// forbidden). If the Go value is nil, then it is illegal for it to be an array -// element, and valueIsNil is returned as true. - -// Returns the TOML type of a Go value. The type may be `nil`, which means -// no concrete TOML type could be found. +// tomlTypeOfGo returns the TOML type name of the Go value's type. +// +// It is used to determine whether the types of array elements are mixed (which +// is forbidden). If the Go value is nil, then it is illegal for it to be an +// array element, and valueIsNil is returned as true. +// +// The type may be `nil`, which means no concrete TOML type could be found. func tomlTypeOfGo(rv reflect.Value) tomlType { if isNil(rv) || !rv.IsValid() { return nil @@ -495,32 +520,43 @@ func tomlTypeOfGo(rv reflect.Value) tomlType { case reflect.Map: return tomlHash case reflect.Struct: - switch rv.Interface().(type) { - case time.Time: + if _, ok := rv.Interface().(time.Time); ok { return tomlDatetime - case encoding.TextMarshaler: + } + if isMarshaler(rv) { return tomlString - default: - // Someone used a pointer receiver: we can make it work for pointer - // values. - if rv.CanAddr() { - _, ok := rv.Addr().Interface().(encoding.TextMarshaler) - if ok { - return tomlString - } - } - return tomlHash } + return tomlHash default: - _, ok := rv.Interface().(encoding.TextMarshaler) - if ok { + if isMarshaler(rv) { return tomlString } + encPanic(errors.New("unsupported type: " + rv.Kind().String())) - panic("") // Need *some* return value + panic("unreachable") } } +func isMarshaler(rv reflect.Value) bool { + switch rv.Interface().(type) { + case encoding.TextMarshaler: + return true + case Marshaler: + return true + } + + // Someone used a pointer receiver: we can make it work for pointer values. + if rv.CanAddr() { + if _, ok := rv.Addr().Interface().(encoding.TextMarshaler); ok { + return true + } + if _, ok := rv.Addr().Interface().(Marshaler); ok { + return true + } + } + return false +} + // tomlArrayType returns the element type of a TOML array. The type returned // may be nil if it cannot be determined (e.g., a nil slice or a zero length // slize). This function may also panic if it finds a type that cannot be @@ -604,7 +640,14 @@ func (enc *Encoder) newline() { // // key = <any value> // -// If inline is true it won't add a newline at the end. +// This is also used for "k = v" in inline tables; so something like this will +// be written in three calls: +// +// ┌────────────────────┐ +// │ ┌───┐ ┌─────┐│ +// v v v v vv +// key = {k = v, k2 = v2} +// func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) { if len(key) == 0 { encPanic(errNoKey) @@ -617,7 +660,8 @@ func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) { } func (enc *Encoder) wf(format string, v ...interface{}) { - if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { + _, err := fmt.Fprintf(enc.w, format, v...) + if err != nil { encPanic(err) } enc.hasWritten = true diff --git a/vendor/github.com/BurntSushi/toml/error.go b/vendor/github.com/BurntSushi/toml/error.go new file mode 100644 index 000000000..36edc4655 --- /dev/null +++ b/vendor/github.com/BurntSushi/toml/error.go @@ -0,0 +1,229 @@ +package toml + +import ( + "fmt" + "strings" +) + +// ParseError is returned when there is an error parsing the TOML syntax. +// +// For example invalid syntax, duplicate keys, etc. +// +// In addition to the error message itself, you can also print detailed location +// information with context by using ErrorWithLocation(): +// +// toml: error: Key 'fruit' was already created and cannot be used as an array. +// +// At line 4, column 2-7: +// +// 2 | fruit = [] +// 3 | +// 4 | [[fruit]] # Not allowed +// ^^^^^ +// +// Furthermore, the ErrorWithUsage() can be used to print the above with some +// more detailed usage guidance: +// +// toml: error: newlines not allowed within inline tables +// +// At line 1, column 18: +// +// 1 | x = [{ key = 42 # +// ^ +// +// Error help: +// +// Inline tables must always be on a single line: +// +// table = {key = 42, second = 43} +// +// It is invalid to split them over multiple lines like so: +// +// # INVALID +// table = { +// key = 42, +// second = 43 +// } +// +// Use regular for this: +// +// [table] +// key = 42 +// second = 43 +type ParseError struct { + Message string // Short technical message. + Usage string // Longer message with usage guidance; may be blank. + Position Position // Position of the error + LastKey string // Last parsed key, may be blank. + Line int // Line the error occurred. Deprecated: use Position. + + err error + input string +} + +// Position of an error. +type Position struct { + Line int // Line number, starting at 1. + Start int // Start of error, as byte offset starting at 0. + Len int // Lenght in bytes. +} + +func (pe ParseError) Error() string { + msg := pe.Message + if msg == "" { // Error from errorf() + msg = pe.err.Error() + } + + if pe.LastKey == "" { + return fmt.Sprintf("toml: line %d: %s", pe.Position.Line, msg) + } + return fmt.Sprintf("toml: line %d (last key %q): %s", + pe.Position.Line, pe.LastKey, msg) +} + +// ErrorWithUsage() returns the error with detailed location context. +// +// See the documentation on ParseError. +func (pe ParseError) ErrorWithPosition() string { + if pe.input == "" { // Should never happen, but just in case. + return pe.Error() + } + + var ( + lines = strings.Split(pe.input, "\n") + col = pe.column(lines) + b = new(strings.Builder) + ) + + msg := pe.Message + if msg == "" { + msg = pe.err.Error() + } + + // TODO: don't show control characters as literals? This may not show up + // well everywhere. + + if pe.Position.Len == 1 { + fmt.Fprintf(b, "toml: error: %s\n\nAt line %d, column %d:\n\n", + msg, pe.Position.Line, col+1) + } else { + fmt.Fprintf(b, "toml: error: %s\n\nAt line %d, column %d-%d:\n\n", + msg, pe.Position.Line, col, col+pe.Position.Len) + } + if pe.Position.Line > 2 { + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-2, lines[pe.Position.Line-3]) + } + if pe.Position.Line > 1 { + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-1, lines[pe.Position.Line-2]) + } + fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line, lines[pe.Position.Line-1]) + fmt.Fprintf(b, "% 10s%s%s\n", "", strings.Repeat(" ", col), strings.Repeat("^", pe.Position.Len)) + return b.String() +} + +// ErrorWithUsage() returns the error with detailed location context and usage +// guidance. +// +// See the documentation on ParseError. +func (pe ParseError) ErrorWithUsage() string { + m := pe.ErrorWithPosition() + if u, ok := pe.err.(interface{ Usage() string }); ok && u.Usage() != "" { + return m + "Error help:\n\n " + + strings.ReplaceAll(strings.TrimSpace(u.Usage()), "\n", "\n ") + + "\n" + } + return m +} + +func (pe ParseError) column(lines []string) int { + var pos, col int + for i := range lines { + ll := len(lines[i]) + 1 // +1 for the removed newline + if pos+ll >= pe.Position.Start { + col = pe.Position.Start - pos + if col < 0 { // Should never happen, but just in case. + col = 0 + } + break + } + pos += ll + } + + return col +} + +type ( + errLexControl struct{ r rune } + errLexEscape struct{ r rune } + errLexUTF8 struct{ b byte } + errLexInvalidNum struct{ v string } + errLexInvalidDate struct{ v string } + errLexInlineTableNL struct{} + errLexStringNL struct{} +) + +func (e errLexControl) Error() string { + return fmt.Sprintf("TOML files cannot contain control characters: '0x%02x'", e.r) +} +func (e errLexControl) Usage() string { return "" } + +func (e errLexEscape) Error() string { return fmt.Sprintf(`invalid escape in string '\%c'`, e.r) } +func (e errLexEscape) Usage() string { return usageEscape } +func (e errLexUTF8) Error() string { return fmt.Sprintf("invalid UTF-8 byte: 0x%02x", e.b) } +func (e errLexUTF8) Usage() string { return "" } +func (e errLexInvalidNum) Error() string { return fmt.Sprintf("invalid number: %q", e.v) } +func (e errLexInvalidNum) Usage() string { return "" } +func (e errLexInvalidDate) Error() string { return fmt.Sprintf("invalid date: %q", e.v) } +func (e errLexInvalidDate) Usage() string { return "" } +func (e errLexInlineTableNL) Error() string { return "newlines not allowed within inline tables" } +func (e errLexInlineTableNL) Usage() string { return usageInlineNewline } +func (e errLexStringNL) Error() string { return "strings cannot contain newlines" } +func (e errLexStringNL) Usage() string { return usageStringNewline } + +const usageEscape = ` +A '\' inside a "-delimited string is interpreted as an escape character. + +The following escape sequences are supported: +\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX + +To prevent a '\' from being recognized as an escape character, use either: + +- a ' or '''-delimited string; escape characters aren't processed in them; or +- write two backslashes to get a single backslash: '\\'. + +If you're trying to add a Windows path (e.g. "C:\Users\martin") then using '/' +instead of '\' will usually also work: "C:/Users/martin". +` + +const usageInlineNewline = ` +Inline tables must always be on a single line: + + table = {key = 42, second = 43} + +It is invalid to split them over multiple lines like so: + + # INVALID + table = { + key = 42, + second = 43 + } + +Use regular for this: + + [table] + key = 42 + second = 43 +` + +const usageStringNewline = ` +Strings must always be on a single line, and cannot span more than one line: + + # INVALID + string = "Hello, + world!" + +Instead use """ or ''' to split strings over multiple lines: + + string = """Hello, + world!""" +` diff --git a/vendor/github.com/BurntSushi/toml/go.sum b/vendor/github.com/BurntSushi/toml/go.sum deleted file mode 100644 index e69de29bb..000000000 --- a/vendor/github.com/BurntSushi/toml/go.sum +++ /dev/null diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index adc4eb5d5..63ef20f47 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -37,28 +37,14 @@ const ( itemInlineTableEnd ) -const ( - eof = 0 - comma = ',' - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' - inlineTableStart = '{' - inlineTableEnd = '}' -) +const eof = 0 type stateFn func(lx *lexer) stateFn +func (p Position) String() string { + return fmt.Sprintf("at line %d; start %d; length %d", p.Line, p.Start, p.Len) +} + type lexer struct { input string start int @@ -67,26 +53,26 @@ type lexer struct { state stateFn items chan item - // Allow for backing up up to four runes. - // This is necessary because TOML contains 3-rune tokens (""" and '''). + // Allow for backing up up to 4 runes. This is necessary because TOML + // contains 3-rune tokens (""" and '''). prevWidths [4]int - nprev int // how many of prevWidths are in use - // If we emit an eof, we can still back up, but it is not OK to call - // next again. - atEOF bool + nprev int // how many of prevWidths are in use + atEOF bool // If we emit an eof, we can still back up, but it is not OK to call next again. // A stack of state functions used to maintain context. - // The idea is to reuse parts of the state machine in various places. - // For example, values can appear at the top level or within arbitrarily - // nested arrays. The last state on the stack is used after a value has - // been lexed. Similarly for comments. + // + // The idea is to reuse parts of the state machine in various places. For + // example, values can appear at the top level or within arbitrarily nested + // arrays. The last state on the stack is used after a value has been lexed. + // Similarly for comments. stack []stateFn } type item struct { - typ itemType - val string - line int + typ itemType + val string + err error + pos Position } func (lx *lexer) nextItem() item { @@ -96,7 +82,7 @@ func (lx *lexer) nextItem() item { return item default: lx.state = lx.state(lx) - //fmt.Printf(" STATE %-24s current: %-10q stack: %s\n", lx.state, lx.current(), lx.stack) + //fmt.Printf(" STATE %-24s current: %-10q stack: %s\n", lx.state, lx.current(), lx.stack) } } } @@ -105,9 +91,9 @@ func lex(input string) *lexer { lx := &lexer{ input: input, state: lexTop, - line: 1, items: make(chan item, 10), stack: make([]stateFn, 0, 10), + line: 1, } return lx } @@ -129,13 +115,25 @@ func (lx *lexer) current() string { return lx.input[lx.start:lx.pos] } +func (lx lexer) getPos() Position { + p := Position{ + Line: lx.line, + Start: lx.start, + Len: lx.pos - lx.start, + } + if p.Len <= 0 { + p.Len = 1 + } + return p +} + func (lx *lexer) emit(typ itemType) { - lx.items <- item{typ, lx.current(), lx.line} + lx.items <- item{typ: typ, pos: lx.getPos(), val: lx.current()} lx.start = lx.pos } func (lx *lexer) emitTrim(typ itemType) { - lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} + lx.items <- item{typ: typ, pos: lx.getPos(), val: strings.TrimSpace(lx.current())} lx.start = lx.pos } @@ -160,7 +158,13 @@ func (lx *lexer) next() (r rune) { r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) if r == utf8.RuneError { - lx.errorf("invalid UTF-8 byte at position %d (line %d): 0x%02x", lx.pos, lx.line, lx.input[lx.pos]) + lx.error(errLexUTF8{lx.input[lx.pos]}) + return utf8.RuneError + } + + // Note: don't use peek() here, as this calls next(). + if isControl(r) || (r == '\r' && (len(lx.input)-1 == lx.pos || lx.input[lx.pos+1] != '\n')) { + lx.errorControlChar(r) return utf8.RuneError } @@ -188,6 +192,7 @@ func (lx *lexer) backup() { lx.prevWidths[1] = lx.prevWidths[2] lx.prevWidths[2] = lx.prevWidths[3] lx.nprev-- + lx.pos -= w if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { lx.line-- @@ -223,18 +228,58 @@ func (lx *lexer) skip(pred func(rune) bool) { } } -// errorf stops all lexing by emitting an error and returning `nil`. +// error stops all lexing by emitting an error and returning `nil`. +// // Note that any value that is a character is escaped if it's a special // character (newlines, tabs, etc.). +func (lx *lexer) error(err error) stateFn { + if lx.atEOF { + return lx.errorPrevLine(err) + } + lx.items <- item{typ: itemError, pos: lx.getPos(), err: err} + return nil +} + +// errorfPrevline is like error(), but sets the position to the last column of +// the previous line. +// +// This is so that unexpected EOF or NL errors don't show on a new blank line. +func (lx *lexer) errorPrevLine(err error) stateFn { + pos := lx.getPos() + pos.Line-- + pos.Len = 1 + pos.Start = lx.pos - 1 + lx.items <- item{typ: itemError, pos: pos, err: err} + return nil +} + +// errorPos is like error(), but allows explicitly setting the position. +func (lx *lexer) errorPos(start, length int, err error) stateFn { + pos := lx.getPos() + pos.Start = start + pos.Len = length + lx.items <- item{typ: itemError, pos: pos, err: err} + return nil +} + +// errorf is like error, and creates a new error. func (lx *lexer) errorf(format string, values ...interface{}) stateFn { - lx.items <- item{ - itemError, - fmt.Sprintf(format, values...), - lx.line, + if lx.atEOF { + pos := lx.getPos() + pos.Line-- + pos.Len = 1 + pos.Start = lx.pos - 1 + lx.items <- item{typ: itemError, pos: pos, err: fmt.Errorf(format, values...)} + return nil } + lx.items <- item{typ: itemError, pos: lx.getPos(), err: fmt.Errorf(format, values...)} return nil } +func (lx *lexer) errorControlChar(cc rune) stateFn { + return lx.errorPos(lx.pos-1, 1, errLexControl{cc}) +} + // lexTop consumes elements at the top level of TOML data. func lexTop(lx *lexer) stateFn { r := lx.next() @@ -242,10 +287,10 @@ func lexTop(lx *lexer) stateFn { return lexSkip(lx, lexTop) } switch r { - case commentStart: + case '#': lx.push(lexTop) return lexCommentStart - case tableStart: + case '[': return lexTableStart case eof: if lx.pos > lx.start { @@ -268,7 +313,7 @@ func lexTop(lx *lexer) stateFn { func lexTopEnd(lx *lexer) stateFn { r := lx.next() switch { - case r == commentStart: + case r == '#': // a comment will read to a newline for us. lx.push(lexTop) return lexCommentStart @@ -292,7 +337,7 @@ func lexTopEnd(lx *lexer) stateFn { // It also handles the case that this is an item in an array of tables. // e.g., '[[name]]'. func lexTableStart(lx *lexer) stateFn { - if lx.peek() == arrayTableStart { + if lx.peek() == '[' { lx.next() lx.emit(itemArrayTableStart) lx.push(lexArrayTableEnd) @@ -309,10 +354,8 @@ func lexTableEnd(lx *lexer) stateFn { } func lexArrayTableEnd(lx *lexer) stateFn { - if r := lx.next(); r != arrayTableEnd { - return lx.errorf( - "expected end of table array name delimiter %q, but got %q instead", - arrayTableEnd, r) + if r := lx.next(); r != ']' { + return lx.errorf("expected end of table array name delimiter ']', but got %q instead", r) } lx.emit(itemArrayTableEnd) return lexTopEnd @@ -321,11 +364,11 @@ func lexArrayTableEnd(lx *lexer) stateFn { func lexTableNameStart(lx *lexer) stateFn { lx.skip(isWhitespace) switch r := lx.peek(); { - case r == tableEnd || r == eof: + case r == ']' || r == eof: return lx.errorf("unexpected end of table name (table names cannot be empty)") - case r == tableSep: + case r == '.': return lx.errorf("unexpected table separator (table names cannot be empty)") - case r == stringStart || r == rawStringStart: + case r == '"' || r == '\'': lx.ignore() lx.push(lexTableNameEnd) return lexQuotedName @@ -342,10 +385,10 @@ func lexTableNameEnd(lx *lexer) stateFn { switch r := lx.next(); { case isWhitespace(r): return lexTableNameEnd - case r == tableSep: + case r == '.': lx.ignore() return lexTableNameStart - case r == tableEnd: + case r == ']': return lx.pop() default: return lx.errorf("expected '.' or ']' to end table name, but got %q instead", r) @@ -379,10 +422,10 @@ func lexQuotedName(lx *lexer) stateFn { switch { case isWhitespace(r): return lexSkip(lx, lexValue) - case r == stringStart: + case r == '"': lx.ignore() // ignore the '"' return lexString - case r == rawStringStart: + case r == '\'': lx.ignore() // ignore the "'" return lexRawString case r == eof: @@ -400,7 +443,7 @@ func lexKeyStart(lx *lexer) stateFn { return lx.errorf("unexpected '=': key name appears blank") case r == '.': return lx.errorf("unexpected '.': keys cannot start with a '.'") - case r == stringStart || r == rawStringStart: + case r == '"' || r == '\'': lx.ignore() fallthrough default: // Bare key @@ -416,7 +459,7 @@ func lexKeyNameStart(lx *lexer) stateFn { return lx.errorf("unexpected '='") case r == '.': return lx.errorf("unexpected '.'") - case r == stringStart || r == rawStringStart: + case r == '"' || r == '\'': lx.ignore() lx.push(lexKeyEnd) return lexQuotedName @@ -434,7 +477,7 @@ func lexKeyEnd(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexKeyEnd) case r == eof: - return lx.errorf("unexpected EOF; expected key separator %q", keySep) + return lx.errorf("unexpected EOF; expected key separator '='") case r == '.': lx.ignore() return lexKeyNameStart @@ -461,17 +504,17 @@ func lexValue(lx *lexer) stateFn { return lexNumberOrDateStart } switch r { - case arrayStart: + case '[': lx.ignore() lx.emit(itemArray) return lexArrayValue - case inlineTableStart: + case '{': lx.ignore() lx.emit(itemInlineTableStart) return lexInlineTableValue - case stringStart: - if lx.accept(stringStart) { - if lx.accept(stringStart) { + case '"': + if lx.accept('"') { + if lx.accept('"') { lx.ignore() // Ignore """ return lexMultilineString } @@ -479,9 +522,9 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the '"' return lexString - case rawStringStart: - if lx.accept(rawStringStart) { - if lx.accept(rawStringStart) { + case '\'': + if lx.accept('\'') { + if lx.accept('\'') { lx.ignore() // Ignore """ return lexMultilineRawString } @@ -520,14 +563,12 @@ func lexArrayValue(lx *lexer) stateFn { switch { case isWhitespace(r) || isNL(r): return lexSkip(lx, lexArrayValue) - case r == commentStart: + case r == '#': lx.push(lexArrayValue) return lexCommentStart - case r == comma: + case r == ',': return lx.errorf("unexpected comma") - case r == arrayEnd: - // NOTE(caleb): The spec isn't clear about whether you can have - // a trailing comma or not, so we'll allow it. + case r == ']': return lexArrayEnd } @@ -540,22 +581,20 @@ func lexArrayValue(lx *lexer) stateFn { // the next value (or the end of the array): it ignores whitespace and newlines // and expects either a ',' or a ']'. func lexArrayValueEnd(lx *lexer) stateFn { - r := lx.next() - switch { + switch r := lx.next(); { case isWhitespace(r) || isNL(r): return lexSkip(lx, lexArrayValueEnd) - case r == commentStart: + case r == '#': lx.push(lexArrayValueEnd) return lexCommentStart - case r == comma: + case r == ',': lx.ignore() return lexArrayValue // move on to the next value - case r == arrayEnd: + case r == ']': return lexArrayEnd + default: + return lx.errorf("expected a comma (',') or array terminator (']'), but got %s", runeOrEOF(r)) } - return lx.errorf( - "expected a comma or array terminator %q, but got %s instead", - arrayEnd, runeOrEOF(r)) } // lexArrayEnd finishes the lexing of an array. @@ -574,13 +613,13 @@ func lexInlineTableValue(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexInlineTableValue) case isNL(r): - return lx.errorf("newlines not allowed within inline tables") - case r == commentStart: + return lx.errorPrevLine(errLexInlineTableNL{}) + case r == '#': lx.push(lexInlineTableValue) return lexCommentStart - case r == comma: + case r == ',': return lx.errorf("unexpected comma") - case r == inlineTableEnd: + case r == '}': return lexInlineTableEnd } lx.backup() @@ -596,23 +635,21 @@ func lexInlineTableValueEnd(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexInlineTableValueEnd) case isNL(r): - return lx.errorf("newlines not allowed within inline tables") - case r == commentStart: + return lx.errorPrevLine(errLexInlineTableNL{}) + case r == '#': lx.push(lexInlineTableValueEnd) return lexCommentStart - case r == comma: + case r == ',': lx.ignore() lx.skip(isWhitespace) if lx.peek() == '}' { return lx.errorf("trailing comma not allowed in inline tables") } return lexInlineTableValue - case r == inlineTableEnd: + case r == '}': return lexInlineTableEnd default: - return lx.errorf( - "expected a comma or an inline table terminator %q, but got %s instead", - inlineTableEnd, runeOrEOF(r)) + return lx.errorf("expected a comma or an inline table terminator '}', but got %s instead", runeOrEOF(r)) } } @@ -638,14 +675,12 @@ func lexString(lx *lexer) stateFn { switch { case r == eof: return lx.errorf(`unexpected EOF; expected '"'`) - case isControl(r) || r == '\r': - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) case isNL(r): - return lx.errorf("strings cannot contain newlines") + return lx.errorPrevLine(errLexStringNL{}) case r == '\\': lx.push(lexString) return lexStringEscape - case r == stringEnd: + case r == '"': lx.backup() lx.emit(itemString) lx.next() @@ -660,23 +695,20 @@ func lexString(lx *lexer) stateFn { func lexMultilineString(lx *lexer) stateFn { r := lx.next() switch r { + default: + return lexMultilineString case eof: return lx.errorf(`unexpected EOF; expected '"""'`) - case '\r': - if lx.peek() != '\n' { - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) - } - return lexMultilineString case '\\': return lexMultilineStringEscape - case stringEnd: + case '"': /// Found " → try to read two more "". - if lx.accept(stringEnd) { - if lx.accept(stringEnd) { + if lx.accept('"') { + if lx.accept('"') { /// Peek ahead: the string can contain " and "", including at the /// end: """str""""" /// 6 or more at the end, however, is an error. - if lx.peek() == stringEnd { + if lx.peek() == '"' { /// Check if we already lexed 5 's; if so we have 6 now, and /// that's just too many man! if strings.HasSuffix(lx.current(), `"""""`) { @@ -699,12 +731,8 @@ func lexMultilineString(lx *lexer) stateFn { } lx.backup() } + return lexMultilineString } - - if isControl(r) { - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) - } - return lexMultilineString } // lexRawString consumes a raw string. Nothing can be escaped in such a string. @@ -712,20 +740,19 @@ func lexMultilineString(lx *lexer) stateFn { func lexRawString(lx *lexer) stateFn { r := lx.next() switch { + default: + return lexRawString case r == eof: return lx.errorf(`unexpected EOF; expected "'"`) - case isControl(r) || r == '\r': - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) case isNL(r): - return lx.errorf("strings cannot contain newlines") - case r == rawStringEnd: + return lx.errorPrevLine(errLexStringNL{}) + case r == '\'': lx.backup() lx.emit(itemRawString) lx.next() lx.ignore() return lx.pop() } - return lexRawString } // lexMultilineRawString consumes a raw string. Nothing can be escaped in such @@ -734,21 +761,18 @@ func lexRawString(lx *lexer) stateFn { func lexMultilineRawString(lx *lexer) stateFn { r := lx.next() switch r { + default: + return lexMultilineRawString case eof: return lx.errorf(`unexpected EOF; expected "'''"`) - case '\r': - if lx.peek() != '\n' { - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) - } - return lexMultilineRawString - case rawStringEnd: + case '\'': /// Found ' → try to read two more ''. - if lx.accept(rawStringEnd) { - if lx.accept(rawStringEnd) { + if lx.accept('\'') { + if lx.accept('\'') { /// Peek ahead: the string can contain ' and '', including at the /// end: '''str''''' /// 6 or more at the end, however, is an error. - if lx.peek() == rawStringEnd { + if lx.peek() == '\'' { /// Check if we already lexed 5 's; if so we have 6 now, and /// that's just too many man! if strings.HasSuffix(lx.current(), "'''''") { @@ -771,12 +795,8 @@ func lexMultilineRawString(lx *lexer) stateFn { } lx.backup() } + return lexMultilineRawString } - - if isControl(r) { - return lx.errorf("control characters are not allowed inside strings: '0x%02x'", r) - } - return lexMultilineRawString } // lexMultilineStringEscape consumes an escaped character. It assumes that the @@ -817,8 +837,7 @@ func lexStringEscape(lx *lexer) stateFn { case 'U': return lexLongUnicodeEscape } - return lx.errorf("invalid escape character %q; only the following escape characters are allowed: "+ - `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) + return lx.error(errLexEscape{r}) } func lexShortUnicodeEscape(lx *lexer) stateFn { @@ -1108,8 +1127,6 @@ func lexComment(lx *lexer) stateFn { lx.backup() lx.emit(itemText) return lx.pop() - case isControl(r): - return lx.errorf("control characters are not allowed inside comments: '0x%02x'", r) default: return lexComment } @@ -1121,52 +1138,6 @@ func lexSkip(lx *lexer, nextState stateFn) stateFn { return nextState } -// isWhitespace returns true if `r` is a whitespace character according -// to the spec. -func isWhitespace(r rune) bool { - return r == '\t' || r == ' ' -} - -func isNL(r rune) bool { - return r == '\n' || r == '\r' -} - -// Control characters except \n, \t -func isControl(r rune) bool { - switch r { - case '\t', '\r', '\n': - return false - default: - return (r >= 0x00 && r <= 0x1f) || r == 0x7f - } -} - -func isDigit(r rune) bool { - return r >= '0' && r <= '9' -} - -func isHexadecimal(r rune) bool { - return (r >= '0' && r <= '9') || - (r >= 'a' && r <= 'f') || - (r >= 'A' && r <= 'F') -} - -func isOctal(r rune) bool { - return r >= '0' && r <= '7' -} - -func isBinary(r rune) bool { - return r == '0' || r == '1' -} - -func isBareKeyChar(r rune) bool { - return (r >= 'A' && r <= 'Z') || - (r >= 'a' && r <= 'z') || - (r >= '0' && r <= '9') || - r == '_' || - r == '-' -} - func (s stateFn) String() string { name := runtime.FuncForPC(reflect.ValueOf(s).Pointer()).Name() if i := strings.LastIndexByte(name, '.'); i > -1 { @@ -1223,3 +1194,26 @@ func (itype itemType) String() string { func (item item) String() string { return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) } + +func isWhitespace(r rune) bool { return r == '\t' || r == ' ' } +func isNL(r rune) bool { return r == '\n' || r == '\r' } +func isControl(r rune) bool { // Control characters except \t, \r, \n + switch r { + case '\t', '\r', '\n': + return false + default: + return (r >= 0x00 && r <= 0x1f) || r == 0x7f + } +} +func isDigit(r rune) bool { return r >= '0' && r <= '9' } +func isBinary(r rune) bool { return r == '0' || r == '1' } +func isOctal(r rune) bool { return r >= '0' && r <= '7' } +func isHexadecimal(r rune) bool { + return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} +func isBareKeyChar(r rune) bool { + return (r >= 'A' && r <= 'Z') || + (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || + r == '_' || r == '-' +} diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/meta.go index ad8899c6c..868619fb9 100644 --- a/vendor/github.com/BurntSushi/toml/decode_meta.go +++ b/vendor/github.com/BurntSushi/toml/meta.go @@ -1,34 +1,39 @@ package toml -import "strings" +import ( + "strings" +) -// MetaData allows access to meta information about TOML data that may not be -// inferable via reflection. In particular, whether a key has been defined and -// the TOML type of a key. +// MetaData allows access to meta information about TOML data that's not +// accessible otherwise. +// +// It allows checking if a key is defined in the TOML data, whether any keys +// were undecoded, and the TOML type of a key. type MetaData struct { + context Key // Used only during decoding. + mapping map[string]interface{} types map[string]tomlType keys []Key - decoded map[string]bool - context Key // Used only during decoding. + decoded map[string]struct{} } // IsDefined reports if the key exists in the TOML data. // // The key should be specified hierarchically, for example to access the TOML -// key "a.b.c" you would use: +// key "a.b.c" you would use IsDefined("a", "b", "c"). Keys are case sensitive. // -// IsDefined("a", "b", "c") -// -// IsDefined will return false if an empty key given. Keys are case sensitive. +// Returns false for an empty key. func (md *MetaData) IsDefined(key ...string) bool { if len(key) == 0 { return false } - var hash map[string]interface{} - var ok bool - var hashOrVal interface{} = md.mapping + var ( + hash map[string]interface{} + ok bool + hashOrVal interface{} = md.mapping + ) for _, k := range key { if hash, ok = hashOrVal.(map[string]interface{}); !ok { return false @@ -45,51 +50,12 @@ func (md *MetaData) IsDefined(key ...string) bool { // Type will return the empty string if given an empty key or a key that does // not exist. Keys are case sensitive. func (md *MetaData) Type(key ...string) string { - fullkey := strings.Join(key, ".") - if typ, ok := md.types[fullkey]; ok { + if typ, ok := md.types[Key(key).String()]; ok { return typ.typeString() } return "" } -// Key represents any TOML key, including key groups. Use (MetaData).Keys to get -// values of this type. -type Key []string - -func (k Key) String() string { return strings.Join(k, ".") } - -func (k Key) maybeQuotedAll() string { - var ss []string - for i := range k { - ss = append(ss, k.maybeQuoted(i)) - } - return strings.Join(ss, ".") -} - -func (k Key) maybeQuoted(i int) string { - if k[i] == "" { - return `""` - } - quote := false - for _, c := range k[i] { - if !isBareKeyChar(c) { - quote = true - break - } - } - if quote { - return `"` + quotedReplacer.Replace(k[i]) + `"` - } - return k[i] -} - -func (k Key) add(piece string) Key { - newKey := make(Key, len(k)+1) - copy(newKey, k) - newKey[len(k)] = piece - return newKey -} - // Keys returns a slice of every key in the TOML data, including key groups. // // Each key is itself a slice, where the first element is the top of the @@ -115,9 +81,40 @@ func (md *MetaData) Keys() []Key { func (md *MetaData) Undecoded() []Key { undecoded := make([]Key, 0, len(md.keys)) for _, key := range md.keys { - if !md.decoded[key.String()] { + if _, ok := md.decoded[key.String()]; !ok { undecoded = append(undecoded, key) } } return undecoded } + +// Key represents any TOML key, including key groups. Use (MetaData).Keys to get +// values of this type. +type Key []string + +func (k Key) String() string { + ss := make([]string, len(k)) + for i := range k { + ss[i] = k.maybeQuoted(i) + } + return strings.Join(ss, ".") +} + +func (k Key) maybeQuoted(i int) string { + if k[i] == "" { + return `""` + } + for _, c := range k[i] { + if !isBareKeyChar(c) { + return `"` + dblQuotedReplacer.Replace(k[i]) + `"` + } + } + return k[i] +} + +func (k Key) add(piece string) Key { + newKey := make(Key, len(k)+1) + copy(newKey, k) + newKey[len(k)] = piece + return newKey +} diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go index d9ae5db94..8269cca17 100644 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -1,7 +1,6 @@ package toml import ( - "errors" "fmt" "strconv" "strings" @@ -12,35 +11,23 @@ import ( ) type parser struct { - mapping map[string]interface{} - types map[string]tomlType - lx *lexer - - ordered []Key // List of keys in the order that they appear in the TOML data. - context Key // Full key for the current hash in scope. - currentKey string // Base key name for everything except hashes. - approxLine int // Rough approximation of line number - implicits map[string]bool // Record implied keys (e.g. 'key.group.names'). -} - -// ParseError is used when a file can't be parsed: for example invalid integer -// literals, duplicate keys, etc. -type ParseError struct { - Message string - Line int - LastKey string -} - -func (pe ParseError) Error() string { - return fmt.Sprintf("Near line %d (last key parsed '%s'): %s", - pe.Line, pe.LastKey, pe.Message) + lx *lexer + context Key // Full key for the current hash in scope. + currentKey string // Base key name for everything except hashes. + pos Position // Current position in the TOML file. + + ordered []Key // List of keys in the order that they appear in the TOML data. + mapping map[string]interface{} // Map keyname → key value. + types map[string]tomlType // Map keyname → TOML type. + implicits map[string]struct{} // Record implicit keys (e.g. "key.group.names"). } func parse(data string) (p *parser, err error) { defer func() { if r := recover(); r != nil { - var ok bool - if err, ok = r.(ParseError); ok { + if pErr, ok := r.(ParseError); ok { + pErr.input = data + err = pErr return } panic(r) @@ -60,8 +47,13 @@ func parse(data string) (p *parser, err error) { if len(data) < 6 { ex = len(data) } - if strings.ContainsRune(data[:ex], 0) { - return nil, errors.New("files cannot contain NULL bytes; probably using UTF-16; TOML files must be UTF-8") + if i := strings.IndexRune(data[:ex], 0); i > -1 { + return nil, ParseError{ + Message: "files cannot contain NULL bytes; probably using UTF-16; TOML files must be UTF-8", + Position: Position{Line: 1, Start: i, Len: 1}, + Line: 1, + input: data, + } } p = &parser{ @@ -69,7 +61,7 @@ func parse(data string) (p *parser, err error) { types: make(map[string]tomlType), lx: lex(data), ordered: make([]Key, 0), - implicits: make(map[string]bool), + implicits: make(map[string]struct{}), } for { item := p.next() @@ -82,12 +74,21 @@ func parse(data string) (p *parser, err error) { return p, nil } +func (p *parser) panicItemf(it item, format string, v ...interface{}) { + panic(ParseError{ + Message: fmt.Sprintf(format, v...), + Position: it.pos, + Line: it.pos.Len, + LastKey: p.current(), + }) +} + func (p *parser) panicf(format string, v ...interface{}) { - msg := fmt.Sprintf(format, v...) panic(ParseError{ - Message: msg, - Line: p.approxLine, - LastKey: p.current(), + Message: fmt.Sprintf(format, v...), + Position: p.pos, + Line: p.pos.Line, + LastKey: p.current(), }) } @@ -95,11 +96,26 @@ func (p *parser) next() item { it := p.lx.nextItem() //fmt.Printf("ITEM %-18s line %-3d │ %q\n", it.typ, it.line, it.val) if it.typ == itemError { - p.panicf("%s", it.val) + if it.err != nil { + panic(ParseError{ + Position: it.pos, + Line: it.pos.Line, + LastKey: p.current(), + err: it.err, + }) + } + + p.panicItemf(it, "%s", it.val) } return it } +func (p *parser) nextPos() item { + it := p.next() + p.pos = it.pos + return it +} + func (p *parser) bug(format string, v ...interface{}) { panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) } @@ -119,11 +135,9 @@ func (p *parser) assertEqual(expected, got itemType) { func (p *parser) topLevel(item item) { switch item.typ { case itemCommentStart: // # .. - p.approxLine = item.line p.expect(itemText) case itemTableStart: // [ .. ] - name := p.next() - p.approxLine = name.line + name := p.nextPos() var key Key for ; name.typ != itemTableEnd && name.typ != itemEOF; name = p.next() { @@ -135,8 +149,7 @@ func (p *parser) topLevel(item item) { p.setType("", tomlHash) p.ordered = append(p.ordered, key) case itemArrayTableStart: // [[ .. ]] - name := p.next() - p.approxLine = name.line + name := p.nextPos() var key Key for ; name.typ != itemArrayTableEnd && name.typ != itemEOF; name = p.next() { @@ -150,8 +163,7 @@ func (p *parser) topLevel(item item) { case itemKeyStart: // key = .. outerContext := p.context /// Read all the key parts (e.g. 'a' and 'b' in 'a.b') - k := p.next() - p.approxLine = k.line + k := p.nextPos() var key Key for ; k.typ != itemKeyEnd && k.typ != itemEOF; k = p.next() { key = append(key, p.keyString(k)) @@ -206,9 +218,9 @@ var datetimeRepl = strings.NewReplacer( func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) { switch it.typ { case itemString: - return p.replaceEscapes(it.val), p.typeOfPrimitive(it) + return p.replaceEscapes(it, it.val), p.typeOfPrimitive(it) case itemMultilineString: - return p.replaceEscapes(stripFirstNewline(stripEscapedNewlines(it.val))), p.typeOfPrimitive(it) + return p.replaceEscapes(it, stripFirstNewline(stripEscapedNewlines(it.val))), p.typeOfPrimitive(it) case itemRawString: return it.val, p.typeOfPrimitive(it) case itemRawMultilineString: @@ -240,10 +252,10 @@ func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) { func (p *parser) valueInteger(it item) (interface{}, tomlType) { if !numUnderscoresOK(it.val) { - p.panicf("Invalid integer %q: underscores must be surrounded by digits", it.val) + p.panicItemf(it, "Invalid integer %q: underscores must be surrounded by digits", it.val) } if numHasLeadingZero(it.val) { - p.panicf("Invalid integer %q: cannot have leading zeroes", it.val) + p.panicItemf(it, "Invalid integer %q: cannot have leading zeroes", it.val) } num, err := strconv.ParseInt(it.val, 0, 64) @@ -254,7 +266,7 @@ func (p *parser) valueInteger(it item) (interface{}, tomlType) { // So mark the former as a bug but the latter as a legitimate user // error. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { - p.panicf("Integer '%s' is out of the range of 64-bit signed integers.", it.val) + p.panicItemf(it, "Integer '%s' is out of the range of 64-bit signed integers.", it.val) } else { p.bug("Expected integer value, but got '%s'.", it.val) } @@ -272,18 +284,18 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) { }) for _, part := range parts { if !numUnderscoresOK(part) { - p.panicf("Invalid float %q: underscores must be surrounded by digits", it.val) + p.panicItemf(it, "Invalid float %q: underscores must be surrounded by digits", it.val) } } if len(parts) > 0 && numHasLeadingZero(parts[0]) { - p.panicf("Invalid float %q: cannot have leading zeroes", it.val) + p.panicItemf(it, "Invalid float %q: cannot have leading zeroes", it.val) } if !numPeriodsOK(it.val) { // As a special case, numbers like '123.' or '1.e2', // which are valid as far as Go/strconv are concerned, // must be rejected because TOML says that a fractional // part consists of '.' followed by 1+ digits. - p.panicf("Invalid float %q: '.' must be followed by one or more digits", it.val) + p.panicItemf(it, "Invalid float %q: '.' must be followed by one or more digits", it.val) } val := strings.Replace(it.val, "_", "", -1) if val == "+nan" || val == "-nan" { // Go doesn't support this, but TOML spec does. @@ -292,9 +304,9 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) { num, err := strconv.ParseFloat(val, 64) if err != nil { if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { - p.panicf("Float '%s' is out of the range of 64-bit IEEE-754 floating-point numbers.", it.val) + p.panicItemf(it, "Float '%s' is out of the range of 64-bit IEEE-754 floating-point numbers.", it.val) } else { - p.panicf("Invalid float value: %q", it.val) + p.panicItemf(it, "Invalid float value: %q", it.val) } } return num, p.typeOfPrimitive(it) @@ -325,7 +337,7 @@ func (p *parser) valueDatetime(it item) (interface{}, tomlType) { } } if !ok { - p.panicf("Invalid TOML Datetime: %q.", it.val) + p.panicItemf(it, "Invalid TOML Datetime: %q.", it.val) } return t, p.typeOfPrimitive(it) } @@ -335,8 +347,12 @@ func (p *parser) valueArray(it item) (interface{}, tomlType) { // p.setType(p.currentKey, typ) var ( - array []interface{} types []tomlType + + // Initialize to a non-nil empty slice. This makes it consistent with + // how S = [] decodes into a non-nil slice inside something like struct + // { S []string }. See #338 + array = []interface{}{} ) for it = p.next(); it.typ != itemArrayEnd; it = p.next() { if it.typ == itemCommentStart { @@ -347,6 +363,12 @@ func (p *parser) valueArray(it item) (interface{}, tomlType) { val, typ := p.value(it, true) array = append(array, val) types = append(types, typ) + + // XXX: types isn't used here, we need it to record the accurate type + // information. + // + // Not entirely sure how to best store this; could use "key[0]", + // "key[1]" notation, or maybe store it on the Array type? } return array, tomlArray } @@ -373,8 +395,7 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom } /// Read all key parts. - k := p.next() - p.approxLine = k.line + k := p.nextPos() var key Key for ; k.typ != itemKeyEnd && k.typ != itemEOF; k = p.next() { key = append(key, p.keyString(k)) @@ -408,7 +429,7 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom // numHasLeadingZero checks if this number has leading zeroes, allowing for '0', // +/- signs, and base prefixes. func numHasLeadingZero(s string) bool { - if len(s) > 1 && s[0] == '0' && isDigit(rune(s[1])) { // >1 to allow "0" and isDigit to allow 0x + if len(s) > 1 && s[0] == '0' && !(s[1] == 'b' || s[1] == 'o' || s[1] == 'x') { // Allow 0b, 0o, 0x return true } if len(s) > 2 && (s[0] == '-' || s[0] == '+') && s[1] == '0' { @@ -503,7 +524,7 @@ func (p *parser) addContext(key Key, array bool) { if hash, ok := hashContext[k].([]map[string]interface{}); ok { hashContext[k] = append(hash, make(map[string]interface{})) } else { - p.panicf("Key '%s' was already created and cannot be used as an array.", keyContext) + p.panicf("Key '%s' was already created and cannot be used as an array.", key) } } else { p.setValue(key[len(key)-1], make(map[string]interface{})) @@ -513,8 +534,8 @@ func (p *parser) addContext(key Key, array bool) { // set calls setValue and setType. func (p *parser) set(key string, val interface{}, typ tomlType) { - p.setValue(p.currentKey, val) - p.setType(p.currentKey, typ) + p.setValue(key, val) + p.setType(key, typ) } // setValue sets the given key to the given value in the current context. @@ -573,27 +594,31 @@ func (p *parser) setValue(key string, value interface{}) { hash[key] = value } -// setType sets the type of a particular value at a given key. -// It should be called immediately AFTER setValue. +// setType sets the type of a particular value at a given key. It should be +// called immediately AFTER setValue. // // Note that if `key` is empty, then the type given will be applied to the // current context (which is either a table or an array of tables). func (p *parser) setType(key string, typ tomlType) { keyContext := make(Key, 0, len(p.context)+1) - for _, k := range p.context { - keyContext = append(keyContext, k) - } + keyContext = append(keyContext, p.context...) if len(key) > 0 { // allow type setting for hashes keyContext = append(keyContext, key) } + // Special case to make empty keys ("" = 1) work. + // Without it it will set "" rather than `""`. + // TODO: why is this needed? And why is this only needed here? + if len(keyContext) == 0 { + keyContext = Key{""} + } p.types[keyContext.String()] = typ } // Implicit keys need to be created when tables are implied in "a.b.c.d = 1" and // "[a.b.c]" (the "a", "b", and "c" hashes are never created explicitly). -func (p *parser) addImplicit(key Key) { p.implicits[key.String()] = true } -func (p *parser) removeImplicit(key Key) { p.implicits[key.String()] = false } -func (p *parser) isImplicit(key Key) bool { return p.implicits[key.String()] } +func (p *parser) addImplicit(key Key) { p.implicits[key.String()] = struct{}{} } +func (p *parser) removeImplicit(key Key) { delete(p.implicits, key.String()) } +func (p *parser) isImplicit(key Key) bool { _, ok := p.implicits[key.String()]; return ok } func (p *parser) isArray(key Key) bool { return p.types[key.String()] == tomlArray } func (p *parser) addImplicitContext(key Key) { p.addImplicit(key) @@ -662,8 +687,8 @@ func stripEscapedNewlines(s string) string { return strings.Join(split, "") } -func (p *parser) replaceEscapes(str string) string { - var replaced []rune +func (p *parser) replaceEscapes(it item, str string) string { + replaced := make([]rune, 0, len(str)) s := []byte(str) r := 0 for r < len(s) { @@ -683,7 +708,7 @@ func (p *parser) replaceEscapes(str string) string { p.bug("Expected valid escape code after \\, but got %q.", s[r]) return "" case ' ', '\t': - p.panicf("invalid escape: '\\%c'", s[r]) + p.panicItemf(it, "invalid escape: '\\%c'", s[r]) return "" case 'b': replaced = append(replaced, rune(0x0008)) @@ -710,14 +735,14 @@ func (p *parser) replaceEscapes(str string) string { // At this point, we know we have a Unicode escape of the form // `uXXXX` at [r, r+5). (Because the lexer guarantees this // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) + escaped := p.asciiEscapeToUnicode(it, s[r+1:r+5]) replaced = append(replaced, escaped) r += 5 case 'U': // At this point, we know we have a Unicode escape of the form // `uXXXX` at [r, r+9). (Because the lexer guarantees this // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) + escaped := p.asciiEscapeToUnicode(it, s[r+1:r+9]) replaced = append(replaced, escaped) r += 9 } @@ -725,15 +750,14 @@ func (p *parser) replaceEscapes(str string) string { return string(replaced) } -func (p *parser) asciiEscapeToUnicode(bs []byte) rune { +func (p *parser) asciiEscapeToUnicode(it item, bs []byte) rune { s := string(bs) hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) if err != nil { - p.bug("Could not parse '%s' as a hexadecimal number, but the "+ - "lexer claims it's OK: %s", s, err) + p.bug("Could not parse '%s' as a hexadecimal number, but the lexer claims it's OK: %s", s, err) } if !utf8.ValidRune(rune(hex)) { - p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) + p.panicItemf(it, "Escaped character '\\u%s' is not valid UTF-8.", s) } return rune(hex) } diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go index 608997c22..254ca82e5 100644 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -70,8 +70,8 @@ func typeFields(t reflect.Type) []field { next := []field{{typ: t}} // Count of queued names for current level and the next. - count := map[reflect.Type]int{} - nextCount := map[reflect.Type]int{} + var count map[reflect.Type]int + var nextCount map[reflect.Type]int // Types already visited at an earlier level. visited := map[reflect.Type]bool{} diff --git a/vendor/github.com/BurntSushi/toml/type_check.go b/vendor/github.com/BurntSushi/toml/type_toml.go index d56aa80fa..4e90d7737 100644 --- a/vendor/github.com/BurntSushi/toml/type_check.go +++ b/vendor/github.com/BurntSushi/toml/type_toml.go @@ -16,7 +16,7 @@ func typeEqual(t1, t2 tomlType) bool { return t1.typeString() == t2.typeString() } -func typeIsHash(t tomlType) bool { +func typeIsTable(t tomlType) bool { return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) } diff --git a/vendor/github.com/containers/common/libimage/pull.go b/vendor/github.com/containers/common/libimage/pull.go index 84ce107ee..ff93b6ed8 100644 --- a/vendor/github.com/containers/common/libimage/pull.go +++ b/vendor/github.com/containers/common/libimage/pull.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "runtime" "strings" "time" @@ -446,12 +447,18 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str // If there's already a local image "localhost/foo", then we should // attempt pulling that instead of doing the full short-name dance. // - // NOTE: we must ignore the platform of a local image when doing - // lookups here, even if arch/os/variant is set. Some images set an - // incorrect or even invalid platform (see containers/podman/issues/10682). - // Doing the lookup while ignoring the platform checks prevents - // redundantly downloading the same image. - localImage, resolvedImageName, err = r.LookupImage(imageName, nil) + // NOTE that we only do platform checks if the specified values differ + // from the local platform. Unfortunately, there are many images used + // in the wild which don't set the correct value(s) in the config + // causing various issues such as containers/podman/issues/10682. + lookupImageOptions := &LookupImageOptions{Variant: options.Variant} + if options.Architecture != runtime.GOARCH { + lookupImageOptions.Architecture = options.Architecture + } + if options.OS != runtime.GOOS { + lookupImageOptions.OS = options.OS + } + localImage, resolvedImageName, err = r.LookupImage(imageName, lookupImageOptions) if err != nil && errors.Cause(err) != storage.ErrImageUnknown { logrus.Errorf("Looking up %s in local storage: %v", imageName, err) } @@ -464,37 +471,18 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str } } - customPlatform := false - if len(options.Architecture)+len(options.OS)+len(options.Variant) > 0 { - customPlatform = true - // Unless the pull policy is "always", we must pessimistically assume - // that the local image has an invalid architecture (see - // containers/podman/issues/10682). Hence, whenever the user requests - // a custom platform, set the pull policy to "always" to make sure - // we're pulling down the image. + customPlatform := len(options.Architecture)+len(options.OS)+len(options.Variant) > 0 + if customPlatform && pullPolicy != config.PullPolicyAlways && pullPolicy != config.PullPolicyNever { + // Unless the pull policy is always/never, we must + // pessimistically assume that the local image has an invalid + // architecture (see containers/podman/issues/10682). Hence, + // whenever the user requests a custom platform, set the pull + // policy to "newer" to make sure we're pulling down the + // correct image. // - // NOTE that this is will even override --pull={false,never}. This is - // very likely a bug but a consistent one in Podman/Buildah and should - // be addressed at a later point. - if pullPolicy != config.PullPolicyAlways && pullPolicy != config.PullPolicyNever { - switch { - // User input clearly refer to a local image. - case strings.HasPrefix(imageName, "localhost/"): - logrus.Debugf("Enforcing pull policy to %q to support custom platform (arch: %q, os: %q, variant: %q)", "never", options.Architecture, options.OS, options.Variant) - pullPolicy = config.PullPolicyNever - - // Image resolved to a local one, so let's still have a - // look at the registries or aliases but use it - // otherwise. - case strings.HasPrefix(resolvedImageName, "localhost/"): - logrus.Debugf("Enforcing pull policy to %q to support custom platform (arch: %q, os: %q, variant: %q)", "newer", options.Architecture, options.OS, options.Variant) - pullPolicy = config.PullPolicyNewer - - default: - logrus.Debugf("Enforcing pull policy to %q to support custom platform (arch: %q, os: %q, variant: %q)", "always", options.Architecture, options.OS, options.Variant) - pullPolicy = config.PullPolicyAlways - } - } + // NOTE that this is will even override --pull={false,never}. + pullPolicy = config.PullPolicyNewer + logrus.Debugf("Enforcing pull policy to %q to pull custom platform (arch: %q, os: %q, variant: %q) - local image may mistakenly specify wrong platform", pullPolicy, options.Architecture, options.OS, options.Variant) } if pullPolicy == config.PullPolicyNever { @@ -540,6 +528,12 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str sys := r.systemContextCopy() resolved, err := shortnames.Resolve(sys, imageName) if err != nil { + // TODO: that is a too big of a hammer since we should only + // ignore errors that indicate that there's no alias and no + // USRs. Must be addressed in c/image first. + if localImage != nil && pullPolicy == config.PullPolicyNewer { + return []string{resolvedImageName}, nil + } return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 9f3270b53..54d49db42 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,7 +1,7 @@ # github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 github.com/Azure/go-ansiterm github.com/Azure/go-ansiterm/winterm -# github.com/BurntSushi/toml v0.4.1 +# github.com/BurntSushi/toml v1.0.0 ## explicit github.com/BurntSushi/toml github.com/BurntSushi/toml/internal @@ -106,7 +106,7 @@ github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/util -# github.com/containers/common v0.46.1-0.20220110152253-5476e2b8dc46 +# github.com/containers/common v0.46.1-0.20220112112017-31e8cc4aeeab ## explicit github.com/containers/common/libimage github.com/containers/common/libimage/manifests |