diff options
Diffstat (limited to 'vendor/github.com')
81 files changed, 4570 insertions, 269 deletions
diff --git a/vendor/github.com/containers/common/libimage/import.go b/vendor/github.com/containers/common/libimage/import.go index 4cce4c9ca..2addfdf98 100644 --- a/vendor/github.com/containers/common/libimage/import.go +++ b/vendor/github.com/containers/common/libimage/import.go @@ -23,6 +23,10 @@ type ImportOptions struct { CommitMessage string // Tag the imported image with this value. Tag string + // Overwrite OS of imported image. + OS string + // Overwrite Arch of imported image. + Arch string } // Import imports a custom tarball at the specified path. Returns the name of @@ -48,8 +52,10 @@ func (r *Runtime) Import(ctx context.Context, path string, options *ImportOption } config := v1.Image{ - Config: ic, - History: hist, + Config: ic, + History: hist, + OS: options.OS, + Architecture: options.Arch, } u, err := url.ParseRequestURI(path) diff --git a/vendor/github.com/containers/common/libimage/pull.go b/vendor/github.com/containers/common/libimage/pull.go index acf5c818f..0271f0051 100644 --- a/vendor/github.com/containers/common/libimage/pull.go +++ b/vendor/github.com/containers/common/libimage/pull.go @@ -351,7 +351,7 @@ func (r *Runtime) copySingleImageFromRegistry(ctx context.Context, imageName str // attempt pulling that instead of doing the full short-name dance. localImage, resolvedImageName, err = r.LookupImage(imageName, nil) if err != nil && errors.Cause(err) != storage.ErrImageUnknown { - return nil, errors.Wrap(err, "error looking up local image") + logrus.Errorf("Looking up %s in local storage: %v", imageName, err) } if pullPolicy == config.PullPolicyNever { diff --git a/vendor/github.com/containers/common/libimage/runtime.go b/vendor/github.com/containers/common/libimage/runtime.go index bbf1c2a61..93a42eb55 100644 --- a/vendor/github.com/containers/common/libimage/runtime.go +++ b/vendor/github.com/containers/common/libimage/runtime.go @@ -132,13 +132,21 @@ func (r *Runtime) storageToImage(storageImage *storage.Image, ref types.ImageRef } // Exists returns true if the specicifed image exists in the local containers -// storage. +// storage. Note that it may return false if an image corrupted. func (r *Runtime) Exists(name string) (bool, error) { image, _, err := r.LookupImage(name, &LookupImageOptions{IgnorePlatform: true}) if err != nil && errors.Cause(err) != storage.ErrImageUnknown { return false, err } - return image != nil, nil + if image == nil { + return false, nil + } + // Inspect the image to make sure if it's corrupted or not. + if _, err := image.Inspect(context.Background(), false); err != nil { + logrus.Errorf("Image %s exists in local storage but may be corrupted: %v", name, err) + return false, nil + } + return true, nil } // LookupImageOptions allow for customizing local image lookups. diff --git a/vendor/github.com/containers/common/pkg/auth/auth.go b/vendor/github.com/containers/common/pkg/auth/auth.go index a9ad60f43..d2e75c1f7 100644 --- a/vendor/github.com/containers/common/pkg/auth/auth.go +++ b/vendor/github.com/containers/common/pkg/auth/auth.go @@ -134,9 +134,13 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO if err = docker.CheckAuth(ctx, systemContext, username, password, server); err == nil { // Write the new credentials to the authfile - if err := config.SetAuthentication(systemContext, server, username, password); err != nil { + desc, err := config.SetCredentials(systemContext, server, username, password) + if err != nil { return err } + if opts.Verbose { + fmt.Fprintln(opts.Stdout, "Used: ", desc) + } } if err == nil { fmt.Fprintln(opts.Stdout, "Login Succeeded!") diff --git a/vendor/github.com/containers/common/pkg/auth/cli.go b/vendor/github.com/containers/common/pkg/auth/cli.go index 1d8eed4ac..5a7c1137c 100644 --- a/vendor/github.com/containers/common/pkg/auth/cli.go +++ b/vendor/github.com/containers/common/pkg/auth/cli.go @@ -20,6 +20,7 @@ type LoginOptions struct { Username string StdinPassword bool GetLoginSet bool + Verbose bool // set to true for verbose output // Options caller can set Stdin io.Reader // set to os.Stdin Stdout io.Writer // set to os.Stdout @@ -47,6 +48,7 @@ func GetLoginFlags(flags *LoginOptions) *pflag.FlagSet { fs.StringVarP(&flags.Username, "username", "u", "", "Username for registry") fs.BoolVar(&flags.StdinPassword, "password-stdin", false, "Take the password from stdin") fs.BoolVar(&flags.GetLoginSet, "get-login", false, "Return the current login user for the registry") + fs.BoolVarP(&flags.Verbose, "verbose", "v", false, "Write more detailed information to stdout") return &fs } diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index edd52f49d..0d23d6ac6 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -5,6 +5,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strings" "sync" @@ -23,7 +24,7 @@ const ( _configPath = "containers/containers.conf" // DefaultContainersConfig holds the default containers config path DefaultContainersConfig = "/usr/share/" + _configPath - // OverrideContainersConfig holds the default config paths overridden by the root user + // OverrideContainersConfig holds the default config path overridden by the root user OverrideContainersConfig = "/etc/" + _configPath // UserOverrideContainersConfig holds the containers config path overridden by the rootless user UserOverrideContainersConfig = ".config/" + _configPath @@ -55,6 +56,8 @@ type Config struct { Engine EngineConfig `toml:"engine"` // Network section defines the configuration of CNI Plugins Network NetworkConfig `toml:"network"` + // Secret section defines configurations for the secret management + Secrets SecretConfig `toml:"secrets"` } // ContainersConfig represents the "containers" TOML config table @@ -137,6 +140,11 @@ type ContainersConfig struct { // Negative values indicate that the log file won't be truncated. LogSizeMax int64 `toml:"log_size_max,omitempty"` + // Specifies default format tag for container log messages. + // This is useful for creating a specific tag for container log messages. + // Containers logs default to truncated container ID as a tag. + LogTag string `toml:"log_tag,omitempty"` + // NetNS indicates how to create a network namespace for the container NetNS string `toml:"netns,omitempty"` @@ -440,6 +448,17 @@ type NetworkConfig struct { NetworkConfigDir string `toml:"network_config_dir,omitempty"` } +// SecretConfig represents the "secret" TOML config table +type SecretConfig struct { + // Driver specifies the secret driver to use. + // Current valid value: + // * file + // * pass + Driver string `toml:"driver,omitempty"` + // Opts contains driver specific options + Opts map[string]string `toml:"opts,omitempty"` +} + // Destination represents destination for remote service type Destination struct { // URI, required. Example: ssh://root@example.com:22/run/podman/podman.sock @@ -513,10 +532,49 @@ func readConfigFromFile(path string, config *Config) error { return nil } +// addConfigs will search one level in the config dirPath for config files +// If the dirPath does not exist, addConfigs will return nil +func addConfigs(dirPath string, configs []string) ([]string, error) { + newConfigs := []string{} + + err := filepath.Walk(dirPath, + // WalkFunc to read additional configs + func(path string, info os.FileInfo, err error) error { + switch { + case err != nil: + // return error (could be a permission problem) + return err + case info == nil: + // this should only happen when err != nil but let's be sure + return nil + case info.IsDir(): + if path != dirPath { + // make sure to not recurse into sub-directories + return filepath.SkipDir + } + // ignore directories + return nil + default: + // only add *.conf files + if strings.HasSuffix(path, ".conf") { + newConfigs = append(newConfigs, path) + } + return nil + } + }, + ) + if os.IsNotExist(err) { + err = nil + } + sort.Strings(newConfigs) + return append(configs, newConfigs...), err +} + // Returns the list of configuration files, if they exist in order of hierarchy. // The files are read in order and each new file can/will override previous // file settings. func systemConfigs() ([]string, error) { + var err error configs := []string{} path := os.Getenv("CONTAINERS_CONF") if path != "" { @@ -531,7 +589,12 @@ func systemConfigs() ([]string, error) { if _, err := os.Stat(OverrideContainersConfig); err == nil { configs = append(configs, OverrideContainersConfig) } - path, err := ifRootlessConfigPath() + configs, err = addConfigs(OverrideContainersConfig+".d", configs) + if err != nil { + return nil, err + } + + path, err = ifRootlessConfigPath() if err != nil { return nil, err } @@ -539,6 +602,10 @@ func systemConfigs() ([]string, error) { if _, err := os.Stat(path); err == nil { configs = append(configs, path) } + configs, err = addConfigs(path+".d", configs) + if err != nil { + return nil, err + } } return configs, nil } diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index d9b379eae..0c8c7532e 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -16,30 +16,16 @@ [containers] -# List of devices. Specified as -# "<device-on-host>:<device-on-container>:<permissions>", for example: -# "/dev/sdc:/dev/xvdc:rwm". -# If it is empty or commented out, only the default devices will be used -# -# devices = [] - -# List of volumes. Specified as -# "<directory-on-host>:<directory-in-container>:<options>", for example: -# "/db:/var/lib/db:ro". -# If it is empty or commented out, no volumes will be added +# List of annotation. Specified as +# "key = value" +# If it is empty or commented out, no annotations will be added # -# volumes = [] +# annotations = [] # Used to change the name of the default AppArmor profile of container engine. # # apparmor_profile = "container-default" -# List of annotation. Specified as -# "key=value" -# If it is empty or commented out, no annotations will be added -# -# annotations = [] - # Default way to to create a cgroup namespace for the container # Options are: # `private` Create private Cgroup Namespace for the container. @@ -93,6 +79,13 @@ default_sysctls = [ # "nofile=1280:2560", # ] +# List of devices. Specified as +# "<device-on-host>:<device-on-container>:<permissions>", for example: +# "/dev/sdc:/dev/xvdc:rwm". +# If it is empty or commented out, only the default devices will be used +# +# devices = [] + # List of default DNS options to be added to /etc/resolv.conf inside of the container. # # dns_options = [] @@ -166,6 +159,12 @@ default_sysctls = [ # # log_size_max = -1 +# Specifies default format tag for container log messages. +# This is useful for creating a specific tag for container log messages. +# Containers logs default to truncated container ID as a tag. +# +# log_tag = "" + # Default way to to create a Network namespace for the container # Options are: # `private` Create private Network Namespace for the container. @@ -179,10 +178,6 @@ default_sysctls = [ # # no_hosts = false -# Maximum number of processes allowed in a container. -# -# pids_limit = 2048 - # Default way to to create a PID namespace for the container # Options are: # `private` Create private PID Namespace for the container. @@ -190,6 +185,13 @@ default_sysctls = [ # # pidns = "private" +# Maximum number of processes allowed in a container. +# +# pids_limit = 2048 + +# Indicates the networking to be used for rootless containers +# rootless_networking = "slirp4netns" + # Path to the seccomp.json profile which is used as the default seccomp profile # for the runtime. # @@ -209,14 +211,7 @@ default_sysctls = [ # Set umask inside the container # -# umask="0022" - -# Default way to to create a UTS namespace for the container -# Options are: -# `private` Create private UTS Namespace for the container. -# `host` Share host UTS Namespace with the container. -# -# utsns = "private" +# umask = "0022" # Default way to to create a User namespace for the container # Options are: @@ -229,7 +224,21 @@ default_sysctls = [ # UIDs are allocated from the "container" UIDs listed in # /etc/subuid & /etc/subgid # -# userns_size=65536 +# userns_size = 65536 + +# Default way to to create a UTS namespace for the container +# Options are: +# `private` Create private UTS Namespace for the container. +# `host` Share host UTS Namespace with the container. +# +# utsns = "private" + +# List of volumes. Specified as +# "<directory-on-host>:<directory-in-container>:<options>", for example: +# "/db:/var/lib/db:ro". +# If it is empty or commented out, no volumes will be added +# +# volumes = [] # The network table contains settings pertaining to the management of # CNI plugins. @@ -254,14 +263,8 @@ default_sysctls = [ # network_config_dir = "/etc/cni/net.d/" [engine] -# Maximum number of image layers to be copied (pulled/pushed) simultaneously. -# Not setting this field, or setting it to zero, will fall back to containers/image defaults. -# image_parallel_copies=0 - -# Manifest Type (oci, v2s2, or v2s1) to use when pulling, pushing, building -# container images. By default image pulled and pushed match the format of the -# source image. Building/committing defaults to OCI. -# image_default_format = "" +# Index to the active service +# active_service = production # Cgroup management implementation used for the runtime. # Valid options "systemd" or "cgroupfs" @@ -319,10 +322,19 @@ default_sysctls = [ # "/usr/share/containers/oci/hooks.d", # ] +# Manifest Type (oci, v2s2, or v2s1) to use when pulling, pushing, building +# container images. By default image pulled and pushed match the format of the +# source image. Building/committing defaults to OCI. +# image_default_format = "" + # Default transport method for pulling and pushing for images # # image_default_transport = "docker://" +# Maximum number of image layers to be copied (pulled/pushed) simultaneously. +# Not setting this field, or setting it to zero, will fall back to containers/image defaults. +# image_parallel_copies = 0 + # Default command to run the infra container # # infra_command = "/pause" @@ -345,7 +357,7 @@ default_sysctls = [ # Indicates if Podman is running inside a VM via Podman Machine. # Podman uses this value to do extra setup around networking from the # container inside the VM to to host. -# machine_enabled=false +# machine_enabled = false # MultiImageArchive - if true, the container engine allows for storing archives # (e.g., of the docker-archive transport) with multiple images. By default, @@ -364,12 +376,12 @@ default_sysctls = [ # Path to the slirp4netns binary # -# network_cmd_path="" +# network_cmd_path = "" # Default options to pass to the slirp4netns binary. # For example "allow_host_loopback=true" # -# network_cmd_options=[] +# network_cmd_options = [] # Whether to use chroot instead of pivot_root in the runtime # @@ -389,27 +401,6 @@ default_sysctls = [ # `podman --remote=true` for access to the remote Podman service. # remote = false -# Indicates the networking to be used for rootless containers -# rootless_networking="slirp4netns" - -# Directory for persistent engine files (database, etc) -# By default, this will be configured relative to where the containers/storage -# stores containers -# Uncomment to change location from this default -# -# static_dir = "/var/lib/containers/storage/libpod" - -# Directory for temporary files. Must be tmpfs (wiped after reboot) -# -# tmp_dir = "/run/libpod" - -# Directory for libpod named volumes. -# By default, this will be configured relative to where containers/storage -# stores containers. -# Uncomment to change location from this default. -# -# volume_path = "/var/lib/containers/storage/volumes" - # Default OCI runtime # # runtime = "crun" @@ -419,20 +410,24 @@ default_sysctls = [ # # runtime_supports_json = ["crun", "runc", "kata", "runsc"] +# List of the OCI runtimes that supports running containers with KVM Separation. +# +# runtime_supports_kvm = ["kata"] + # List of the OCI runtimes that supports running containers without cgroups. # # runtime_supports_nocgroups = ["crun"] -# List of the OCI runtimes that supports running containers with KVM Separation. +# Directory for persistent engine files (database, etc) +# By default, this will be configured relative to where the containers/storage +# stores containers +# Uncomment to change location from this default # -# runtime_supports_kvm = ["kata"] +# static_dir = "/var/lib/containers/storage/libpod" # Number of seconds to wait for container to exit before sending kill signal. # stop_timeout = 10 -# Index to the active service -# active_service = production - # map of service destinations # [service_destinations] # [service_destinations.production] @@ -442,10 +437,21 @@ default_sysctls = [ # rootfull "unix://run/podman/podman.sock (Default) # remote rootless ssh://engineering.lab.company.com/run/user/1000/podman/podman.sock # remote rootfull ssh://root@10.10.1.136:22/run/podman/podman.sock -# uri="ssh://user@production.example.com/run/user/1001/podman/podman.sock" +# uri = "ssh://user@production.example.com/run/user/1001/podman/podman.sock" # Path to file containing ssh identity key # identity = "~/.ssh/id_rsa" +# Directory for temporary files. Must be tmpfs (wiped after reboot) +# +# tmp_dir = "/run/libpod" + +# Directory for libpod named volumes. +# By default, this will be configured relative to where containers/storage +# stores containers. +# Uncomment to change location from this default. +# +# volume_path = "/var/lib/containers/storage/volumes" + # Paths to look for a valid OCI runtime (crun, runc, kata, runsc, etc) [engine.runtimes] # crun = [ @@ -458,16 +464,6 @@ default_sysctls = [ # "/run/current-system/sw/bin/crun", # ] -# runc = [ -# "/usr/bin/runc", -# "/usr/sbin/runc", -# "/usr/local/bin/runc", -# "/usr/local/sbin/runc", -# "/sbin/runc", -# "/bin/runc", -# "/usr/lib/cri-o-runc/sbin/runc", -# ] - # kata = [ # "/usr/bin/kata-runtime", # "/usr/sbin/kata-runtime", @@ -479,6 +475,16 @@ default_sysctls = [ # "/usr/bin/kata-fc", # ] +# runc = [ +# "/usr/bin/runc", +# "/usr/sbin/runc", +# "/usr/local/bin/runc", +# "/usr/local/sbin/runc", +# "/sbin/runc", +# "/bin/runc", +# "/usr/lib/cri-o-runc/sbin/runc", +# ] + # runsc = [ # "/usr/bin/runsc", # "/usr/sbin/runsc", @@ -497,3 +503,9 @@ default_sysctls = [ # TOML does not provide a way to end a table other than a further table being # defined, so every key hereafter will be part of [volume_plugins] and not the # main config. + +[secret] +# driver = "file" + +[secret.opts] +# root = "/example/directory" diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index d5a7d5b84..5abb6326f 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -144,8 +144,6 @@ func DefaultConfig() (*Config, error) { return nil, err } - netns := "bridge" - cniConfig := _cniConfigDir defaultEngineConfig.SignaturePolicyPath = DefaultSignaturePolicyPath @@ -161,7 +159,6 @@ func DefaultConfig() (*Config, error) { defaultEngineConfig.SignaturePolicyPath = DefaultSignaturePolicyPath } } - netns = "slirp4netns" cniConfig = filepath.Join(configHome, _cniConfigDirRootless) } @@ -197,12 +194,10 @@ func DefaultConfig() (*Config, error) { IPCNS: "private", LogDriver: defaultLogDriver(), LogSizeMax: DefaultLogSizeMax, - NetNS: netns, NoHosts: false, PidsLimit: DefaultPidsLimit, PidNS: "private", RootlessNetworking: DefaultRootlessNetwork, - SeccompProfile: SeccompDefaultPath, ShmSize: DefaultShmSize, TZ: "", Umask: "0022", @@ -216,10 +211,19 @@ func DefaultConfig() (*Config, error) { NetworkConfigDir: cniConfig, CNIPluginDirs: cniBinDir, }, - Engine: *defaultEngineConfig, + Engine: *defaultEngineConfig, + Secrets: defaultSecretConfig(), }, nil } +// defaultSecretConfig returns the default secret configuration. +// Please note that the default is choosing the "file" driver. +func defaultSecretConfig() SecretConfig { + return SecretConfig{ + Driver: "file", + } +} + // defaultConfigFromMemory returns a default engine configuration. Note that the // config is different for root and rootless. It also parses the storage.conf. func defaultConfigFromMemory() (*EngineConfig, error) { diff --git a/vendor/github.com/containers/common/pkg/report/template.go b/vendor/github.com/containers/common/pkg/report/template.go index f7b4506bb..f86b07034 100644 --- a/vendor/github.com/containers/common/pkg/report/template.go +++ b/vendor/github.com/containers/common/pkg/report/template.go @@ -130,7 +130,7 @@ func NewTemplate(name string) *Template { func (t *Template) Parse(text string) (*Template, error) { if strings.HasPrefix(text, "table ") { t.isTable = true - text = "{{range .}}" + NormalizeFormat(text) + "{{end}}" + text = "{{range .}}" + NormalizeFormat(text) + "{{end -}}" } else { text = NormalizeFormat(text) } @@ -157,12 +157,12 @@ func (t *Template) IsTable() bool { return t.isTable } -var rangeRegex = regexp.MustCompile(`{{\s*range\s*\.\s*}}.*{{\s*end\s*}}`) +var rangeRegex = regexp.MustCompile(`{{\s*range\s*\.\s*}}.*{{\s*end\s*-?\s*}}`) // EnforceRange ensures that the format string contains a range func EnforceRange(format string) string { if !rangeRegex.MatchString(format) { - return "{{range .}}" + format + "{{end}}" + return "{{range .}}" + format + "{{end -}}" } return format } diff --git a/vendor/github.com/containers/common/pkg/seccomp/conversion.go b/vendor/github.com/containers/common/pkg/seccomp/conversion.go index dfab381a5..4c25cb1b1 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/conversion.go +++ b/vendor/github.com/containers/common/pkg/seccomp/conversion.go @@ -118,6 +118,7 @@ func specToSeccomp(spec *specs.LinuxSeccomp) (*Seccomp, error) { return nil, errors.Wrap(err, "convert default action") } res.DefaultAction = newDefaultAction + res.DefaultErrnoRet = spec.DefaultErrnoRet // Loop through all syscall blocks and convert them to the internal format for _, call := range spec.Syscalls { diff --git a/vendor/github.com/containers/common/pkg/seccomp/default_linux.go b/vendor/github.com/containers/common/pkg/seccomp/default_linux.go index f86f3e2ba..edb1294d6 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/default_linux.go +++ b/vendor/github.com/containers/common/pkg/seccomp/default_linux.go @@ -44,10 +44,56 @@ func arches() []Architecture { // DefaultProfile defines the allowlist for the default seccomp profile. func DefaultProfile() *Seccomp { einval := uint(unix.EINVAL) + enosys := uint(unix.ENOSYS) + eperm := uint(unix.EPERM) syscalls := []*Syscall{ { Names: []string{ + "bdflush", + "clone3", + "io_pgetevents", + "io_uring_enter", + "io_uring_register", + "io_uring_setup", + "kexec_file_load", + "kexec_load", + "membarrier", + "migrate_pages", + "move_pages", + "nfsservctl", + "nice", + "oldfstat", + "oldlstat", + "oldolduname", + "oldstat", + "olduname", + "pciconfig_iobase", + "pciconfig_read", + "pciconfig_write", + "pkey_alloc", + "pkey_free", + "pkey_mprotect", + "rseq", + "sgetmask", + "ssetmask", + "swapcontext", + "swapoff", + "swapon", + "sysfs", + "uselib", + "userfaultfd", + "ustat", + "vm86", + "vm86old", + "vmsplice", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + }, + { + Names: []string{ "_llseek", "_newselect", "accept", @@ -128,6 +174,7 @@ func DefaultProfile() *Seccomp { "ftruncate", "ftruncate64", "futex", + "futex_time64", "futimesat", "get_robust_list", "get_thread_area", @@ -212,7 +259,9 @@ func DefaultProfile() *Seccomp { "mq_notify", "mq_open", "mq_timedreceive", + "mq_timedreceive_time64", "mq_timedsend", + "mq_timedsend_time64", "mq_unlink", "mremap", "msgctl", @@ -252,6 +301,7 @@ func DefaultProfile() *Seccomp { "pwritev2", "read", "readahead", + "readdir", "readlink", "readlinkat", "readv", @@ -259,6 +309,7 @@ func DefaultProfile() *Seccomp { "recv", "recvfrom", "recvmmsg", + "recvmmsg_time64", "recvmsg", "remap_file_pages", "removexattr", @@ -274,6 +325,7 @@ func DefaultProfile() *Seccomp { "rt_sigreturn", "rt_sigsuspend", "rt_sigtimedwait", + "rt_sigtimedwait_time64", "rt_tgsigqueueinfo", "sched_get_priority_max", "sched_get_priority_min", @@ -282,6 +334,7 @@ func DefaultProfile() *Seccomp { "sched_getparam", "sched_getscheduler", "sched_rr_get_interval", + "sched_rr_get_interval_time64", "sched_setaffinity", "sched_setattr", "sched_setparam", @@ -293,6 +346,7 @@ func DefaultProfile() *Seccomp { "semget", "semop", "semtimedop", + "semtimedop_time64", "send", "sendfile", "sendfile64", @@ -361,6 +415,7 @@ func DefaultProfile() *Seccomp { "timer_gettime", "timer_gettime64", "timer_settime", + "timer_settime64", "timerfd_create", "timerfd_gettime", "timerfd_gettime64", @@ -516,6 +571,17 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "open_by_handle_at", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_DAC_READ_SEARCH"}, + }, + }, + { + Names: []string{ "bpf", "fanotify_init", "lookup_dcookie", @@ -533,6 +599,24 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "bpf", + "fanotify_init", + "lookup_dcookie", + "perf_event_open", + "quotactl", + "setdomainname", + "sethostname", + "setns", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_ADMIN"}, + }, + }, + { + Names: []string{ "chroot", }, Action: ActAllow, @@ -543,6 +627,17 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "chroot", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_CHROOT"}, + }, + }, + { + Names: []string{ "delete_module", "init_module", "finit_module", @@ -556,6 +651,20 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "delete_module", + "init_module", + "finit_module", + "query_module", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_MODULE"}, + }, + }, + { + Names: []string{ "get_mempolicy", "mbind", "set_mempolicy", @@ -568,6 +677,19 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "get_mempolicy", + "mbind", + "set_mempolicy", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_NICE"}, + }, + }, + { + Names: []string{ "acct", }, Action: ActAllow, @@ -578,6 +700,17 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "acct", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_PACCT"}, + }, + }, + { + Names: []string{ "kcmp", "process_madvise", "process_vm_readv", @@ -592,6 +725,21 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "kcmp", + "process_madvise", + "process_vm_readv", + "process_vm_writev", + "ptrace", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_PTRACE"}, + }, + }, + { + Names: []string{ "iopl", "ioperm", }, @@ -603,6 +751,18 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "iopl", + "ioperm", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_RAWIO"}, + }, + }, + { + Names: []string{ "settimeofday", "stime", "clock_settime", @@ -616,6 +776,20 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "settimeofday", + "stime", + "clock_settime", + "clock_settime64", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_TIME"}, + }, + }, + { + Names: []string{ "vhangup", }, Action: ActAllow, @@ -626,6 +800,17 @@ func DefaultProfile() *Seccomp { }, { Names: []string{ + "vhangup", + }, + Action: ActErrno, + ErrnoRet: &eperm, + Args: []*Arg{}, + Excludes: Filter{ + Caps: []string{"CAP_SYS_TTY_CONFIG"}, + }, + }, + { + Names: []string{ "socket", }, Action: ActErrno, @@ -706,8 +891,9 @@ func DefaultProfile() *Seccomp { } return &Seccomp{ - DefaultAction: ActErrno, - ArchMap: arches(), - Syscalls: syscalls, + DefaultAction: ActErrno, + DefaultErrnoRet: &enosys, + ArchMap: arches(), + Syscalls: syscalls, } } diff --git a/vendor/github.com/containers/common/pkg/seccomp/filter.go b/vendor/github.com/containers/common/pkg/seccomp/filter.go index ac9b2698f..90da99f0a 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/filter.go +++ b/vendor/github.com/containers/common/pkg/seccomp/filter.go @@ -41,7 +41,7 @@ func BuildFilter(spec *specs.LinuxSeccomp) (*libseccomp.ScmpFilter, error) { return nil, errors.Wrap(err, "convert spec to seccomp profile") } - defaultAction, err := toAction(profile.DefaultAction, nil) + defaultAction, err := toAction(profile.DefaultAction, profile.DefaultErrnoRet) if err != nil { return nil, errors.Wrapf(err, "convert default action %s", profile.DefaultAction) } diff --git a/vendor/github.com/containers/common/pkg/seccomp/seccomp.json b/vendor/github.com/containers/common/pkg/seccomp/seccomp.json index 8d799fd02..885240e50 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/seccomp.json +++ b/vendor/github.com/containers/common/pkg/seccomp/seccomp.json @@ -1,5 +1,6 @@ { "defaultAction": "SCMP_ACT_ERRNO", + "defaultErrnoRet": 38, "archMap": [ { "architecture": "SCMP_ARCH_X86_64", @@ -52,6 +53,53 @@ "syscalls": [ { "names": [ + "bdflush", + "clone3", + "io_pgetevents", + "io_uring_enter", + "io_uring_register", + "io_uring_setup", + "kexec_file_load", + "kexec_load", + "membarrier", + "migrate_pages", + "move_pages", + "nfsservctl", + "nice", + "oldfstat", + "oldlstat", + "oldolduname", + "oldstat", + "olduname", + "pciconfig_iobase", + "pciconfig_read", + "pciconfig_write", + "pkey_alloc", + "pkey_free", + "pkey_mprotect", + "rseq", + "sgetmask", + "ssetmask", + "swapcontext", + "swapoff", + "swapon", + "sysfs", + "uselib", + "userfaultfd", + "ustat", + "vm86", + "vm86old", + "vmsplice" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": {}, + "errnoRet": 1 + }, + { + "names": [ "_llseek", "_newselect", "accept", @@ -132,6 +180,7 @@ "ftruncate", "ftruncate64", "futex", + "futex_time64", "futimesat", "get_robust_list", "get_thread_area", @@ -216,7 +265,9 @@ "mq_notify", "mq_open", "mq_timedreceive", + "mq_timedreceive_time64", "mq_timedsend", + "mq_timedsend_time64", "mq_unlink", "mremap", "msgctl", @@ -256,6 +307,7 @@ "pwritev2", "read", "readahead", + "readdir", "readlink", "readlinkat", "readv", @@ -263,6 +315,7 @@ "recv", "recvfrom", "recvmmsg", + "recvmmsg_time64", "recvmsg", "remap_file_pages", "removexattr", @@ -278,6 +331,7 @@ "rt_sigreturn", "rt_sigsuspend", "rt_sigtimedwait", + "rt_sigtimedwait_time64", "rt_tgsigqueueinfo", "sched_get_priority_max", "sched_get_priority_min", @@ -286,6 +340,7 @@ "sched_getparam", "sched_getscheduler", "sched_rr_get_interval", + "sched_rr_get_interval_time64", "sched_setaffinity", "sched_setattr", "sched_setparam", @@ -297,6 +352,7 @@ "semget", "semop", "semtimedop", + "semtimedop_time64", "send", "sendfile", "sendfile64", @@ -365,6 +421,7 @@ "timer_gettime", "timer_gettime64", "timer_settime", + "timer_settime64", "timerfd_create", "timerfd_gettime", "timerfd_gettime64", @@ -582,6 +639,21 @@ }, { "names": [ + "open_by_handle_at" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_DAC_READ_SEARCH" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "bpf", "fanotify_init", "lookup_dcookie", @@ -603,6 +675,28 @@ }, { "names": [ + "bpf", + "fanotify_init", + "lookup_dcookie", + "perf_event_open", + "quotactl", + "setdomainname", + "sethostname", + "setns" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_ADMIN" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "chroot" ], "action": "SCMP_ACT_ALLOW", @@ -617,6 +711,21 @@ }, { "names": [ + "chroot" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_CHROOT" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "delete_module", "init_module", "finit_module", @@ -634,6 +743,24 @@ }, { "names": [ + "delete_module", + "init_module", + "finit_module", + "query_module" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_MODULE" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "get_mempolicy", "mbind", "set_mempolicy" @@ -650,6 +777,23 @@ }, { "names": [ + "get_mempolicy", + "mbind", + "set_mempolicy" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_NICE" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "acct" ], "action": "SCMP_ACT_ALLOW", @@ -664,6 +808,21 @@ }, { "names": [ + "acct" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_PACCT" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "kcmp", "process_madvise", "process_vm_readv", @@ -682,6 +841,25 @@ }, { "names": [ + "kcmp", + "process_madvise", + "process_vm_readv", + "process_vm_writev", + "ptrace" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_PTRACE" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "iopl", "ioperm" ], @@ -697,6 +875,22 @@ }, { "names": [ + "iopl", + "ioperm" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_RAWIO" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "settimeofday", "stime", "clock_settime", @@ -714,6 +908,24 @@ }, { "names": [ + "settimeofday", + "stime", + "clock_settime", + "clock_settime64" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_TIME" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "vhangup" ], "action": "SCMP_ACT_ALLOW", @@ -728,6 +940,21 @@ }, { "names": [ + "vhangup" + ], + "action": "SCMP_ACT_ERRNO", + "args": [], + "comment": "", + "includes": {}, + "excludes": { + "caps": [ + "CAP_SYS_TTY_CONFIG" + ] + }, + "errnoRet": 1 + }, + { + "names": [ "socket" ], "action": "SCMP_ACT_ERRNO", diff --git a/vendor/github.com/containers/common/pkg/seccomp/seccomp_linux.go b/vendor/github.com/containers/common/pkg/seccomp/seccomp_linux.go index 19500cc97..af36b9990 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/seccomp_linux.go +++ b/vendor/github.com/containers/common/pkg/seccomp/seccomp_linux.go @@ -111,6 +111,7 @@ func setupSeccomp(config *Seccomp, rs *specs.Spec) (*specs.LinuxSeccomp, error) } newConfig.DefaultAction = specs.LinuxSeccompAction(config.DefaultAction) + newConfig.DefaultErrnoRet = config.DefaultErrnoRet Loop: // Loop through all syscall blocks and convert them to libcontainer format after filtering them diff --git a/vendor/github.com/containers/common/pkg/seccomp/types.go b/vendor/github.com/containers/common/pkg/seccomp/types.go index 7b0436dfc..36712458a 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/types.go +++ b/vendor/github.com/containers/common/pkg/seccomp/types.go @@ -6,7 +6,8 @@ package seccomp // Seccomp represents the config for a seccomp profile for syscall restriction. type Seccomp struct { - DefaultAction Action `json:"defaultAction"` + DefaultAction Action `json:"defaultAction"` + DefaultErrnoRet *uint `json:"defaultErrnoRet"` // Architectures is kept to maintain backward compatibility with the old // seccomp profile. Architectures []Arch `json:"architectures,omitempty"` diff --git a/vendor/github.com/containers/common/pkg/secrets/passdriver/passdriver.go b/vendor/github.com/containers/common/pkg/secrets/passdriver/passdriver.go new file mode 100644 index 000000000..6dc00b34c --- /dev/null +++ b/vendor/github.com/containers/common/pkg/secrets/passdriver/passdriver.go @@ -0,0 +1,176 @@ +package passdriver + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/pkg/errors" +) + +var ( + // errNoSecretData indicates that there is not data associated with an id + errNoSecretData = errors.New("no secret data with ID") + + // errNoSecretData indicates that there is secret data already associated with an id + errSecretIDExists = errors.New("secret data with ID already exists") + + // errInvalidKey indicates that something about your key is wrong + errInvalidKey = errors.New("invalid key") +) + +type driverConfig struct { + // Root contains the root directory where the secrets are stored + Root string + // KeyID contains the key id that will be used for encryption (i.e. user@domain.tld) + KeyID string +} + +func (cfg *driverConfig) ParseOpts(opts map[string]string) { + if val, ok := opts["root"]; ok { + cfg.Root = val + cfg.findGpgID() // try to find a .gpg-id in the parent directories of Root + } + if val, ok := opts["key"]; ok { + cfg.KeyID = val + } +} + +func defaultDriverConfig() *driverConfig { + cfg := &driverConfig{} + + if home, err := os.UserHomeDir(); err == nil { + defaultLocations := []string{ + filepath.Join(home, ".password-store"), + filepath.Join(home, ".local/share/gopass/stores/root"), + } + for _, path := range defaultLocations { + if stat, err := os.Stat(path); err != nil || !stat.IsDir() { + continue + } + cfg.Root = path + bs, err := ioutil.ReadFile(filepath.Join(path, ".gpg-id")) + if err != nil { + continue + } + cfg.KeyID = string(bytes.Trim(bs, "\r\n")) + break + } + } + + return cfg +} + +func (cfg *driverConfig) findGpgID() { + path := cfg.Root + for len(path) > 1 { + if _, err := os.Stat(filepath.Join(path, ".gpg-id")); err == nil { + bs, err := ioutil.ReadFile(filepath.Join(path, ".gpg-id")) + if err != nil { + continue + } + cfg.KeyID = string(bytes.Trim(bs, "\r\n")) + break + } + path = filepath.Dir(path) + } +} + +// Driver is the passdriver object +type Driver struct { + driverConfig +} + +// NewDriver creates a new secret driver. +func NewDriver(opts map[string]string) (*Driver, error) { + cfg := defaultDriverConfig() + cfg.ParseOpts(opts) + + driver := &Driver{ + driverConfig: *cfg, + } + + return driver, nil +} + +// List returns all secret IDs +func (d *Driver) List() (secrets []string, err error) { + files, err := ioutil.ReadDir(d.Root) + if err != nil { + return nil, errors.Wrap(err, "failed to read secret directory") + } + for _, f := range files { + fileName := f.Name() + withoutSuffix := fileName[:len(fileName)-len(".gpg")] + secrets = append(secrets, withoutSuffix) + } + sort.Strings(secrets) + return secrets, nil +} + +// Lookup returns the bytes associated with a secret ID +func (d *Driver) Lookup(id string) ([]byte, error) { + out := &bytes.Buffer{} + key, err := d.getPath(id) + if err != nil { + return nil, err + } + if err := d.gpg(context.TODO(), nil, out, "--decrypt", key); err != nil { + return nil, errors.Wrapf(errNoSecretData, id) + } + if out.Len() == 0 { + return nil, errors.Wrapf(errNoSecretData, id) + } + return out.Bytes(), nil +} + +// Store saves the bytes associated with an ID. An error is returned if the ID already exists +func (d *Driver) Store(id string, data []byte) error { + if _, err := d.Lookup(id); err == nil { + return errors.Wrap(errSecretIDExists, id) + } + in := bytes.NewReader(data) + key, err := d.getPath(id) + if err != nil { + return err + } + return d.gpg(context.TODO(), in, nil, "--encrypt", "-r", d.KeyID, "-o", key) +} + +// Delete removes the secret associated with the specified ID. An error is returned if no matching secret is found. +func (d *Driver) Delete(id string) error { + key, err := d.getPath(id) + if err != nil { + return err + } + if err := os.Remove(key); err != nil { + return errors.Wrap(errNoSecretData, id) + } + return nil +} + +func (d *Driver) gpg(ctx context.Context, in io.Reader, out io.Writer, args ...string) error { + cmd := exec.CommandContext(ctx, "gpg", args...) + cmd.Env = os.Environ() + cmd.Stdin = in + cmd.Stdout = out + cmd.Stderr = ioutil.Discard + return cmd.Run() +} + +func (d *Driver) getPath(id string) (string, error) { + path, err := filepath.Abs(filepath.Join(d.Root, id)) + if err != nil { + return "", errInvalidKey + } + if !strings.HasPrefix(path, d.Root) { + return "", errInvalidKey + } + return path + ".gpg", nil +} diff --git a/vendor/github.com/containers/common/pkg/secrets/secrets.go b/vendor/github.com/containers/common/pkg/secrets/secrets.go index d27bb7472..2573b80e2 100644 --- a/vendor/github.com/containers/common/pkg/secrets/secrets.go +++ b/vendor/github.com/containers/common/pkg/secrets/secrets.go @@ -8,6 +8,7 @@ import ( "time" "github.com/containers/common/pkg/secrets/filedriver" + "github.com/containers/common/pkg/secrets/passdriver" "github.com/containers/storage/pkg/lockfile" "github.com/containers/storage/pkg/stringid" "github.com/pkg/errors" @@ -271,12 +272,15 @@ func validateSecretName(name string) error { // getDriver creates a new driver. func getDriver(name string, opts map[string]string) (SecretsDriver, error) { - if name == "file" { + switch name { + case "file": if path, ok := opts["path"]; ok { return filedriver.NewDriver(path) } else { return nil, errors.Wrap(errInvalidDriverOpt, "need path for filedriver") } + case "pass": + return passdriver.NewDriver(opts) } return nil, errInvalidDriver } diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index ff3bf9a32..e8e3d72fe 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.39.1-dev" +const Version = "0.40.0" diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index ed76283f9..62f47c013 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -29,10 +29,10 @@ import ( imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/vbauerster/mpb/v6" - "github.com/vbauerster/mpb/v6/decor" - "golang.org/x/crypto/ssh/terminal" + "github.com/vbauerster/mpb/v7" + "github.com/vbauerster/mpb/v7/decor" "golang.org/x/sync/semaphore" + "golang.org/x/term" ) type digestingReader struct { @@ -43,10 +43,6 @@ type digestingReader struct { validationSucceeded bool } -// FIXME: disable early layer commits temporarily until a solid solution to -// address #1205 has been found. -const enableEarlyCommit = false - var ( // ErrDecryptParamsMissing is returned if there is missing decryption parameters ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present") @@ -864,7 +860,7 @@ func (ic *imageCopier) noPendingManifestUpdates() bool { // isTTY returns true if the io.Writer is a file and a tty. func isTTY(w io.Writer) bool { if f, ok := w.(*os.File); ok { - return terminal.IsTerminal(int(f.Fd())) + return term.IsTerminal(int(f.Fd())) } return false } @@ -893,6 +889,18 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { err error } + // The manifest is used to extract the information whether a given + // layer is empty. + manifestBlob, manifestType, err := ic.src.Manifest(ctx) + if err != nil { + return err + } + man, err := manifest.FromBlob(manifestBlob, manifestType) + if err != nil { + return err + } + manifestLayerInfos := man.LayerInfos() + // copyGroup is used to determine if all layers are copied copyGroup := sync.WaitGroup{} @@ -925,7 +933,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name()) } } else { - cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index, srcRef) + cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index, srcRef, manifestLayerInfos[index].EmptyLayer) } data[index] = cld } @@ -1094,8 +1102,9 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind ), ) } else { + sstyle := mpb.SpinnerStyle(".", "..", "...", "....", "").PositionLeft() bar = pool.Add(0, - mpb.NewSpinnerFiller([]string{".", "..", "...", "....", ""}, mpb.SpinnerOnLeft), + sstyle.Build(), mpb.BarFillerClearOnComplete(), mpb.PrependDecorators( decor.OnComplete(decor.Name(prefix), onComplete), @@ -1121,7 +1130,7 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error { progressPool, progressCleanup := c.newProgressPool(ctx) defer progressCleanup() bar := c.createProgressBar(progressPool, srcInfo, "config", "done") - destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1) + destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1, false) if err != nil { return types.BlobInfo{}, err } @@ -1148,7 +1157,7 @@ type diffIDResult struct { // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded // srcRef can be used as an additional hint to the destination during checking whehter a layer can be reused but srcRef can be nil. -func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int, srcRef reference.Named) (types.BlobInfo, digest.Digest, error) { +func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int, srcRef reference.Named, emptyLayer bool) (types.BlobInfo, digest.Digest, error) { // If the srcInfo doesn't contain compression information, try to compute it from the // MediaType, which was either read from a manifest by way of LayerInfos() or constructed // by LayerInfosForCopy(), if it was supplied at all. If we succeed in copying the blob, @@ -1195,10 +1204,9 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to Cache: ic.c.blobInfoCache, CanSubstitute: ic.canSubstituteBlobs, SrcRef: srcRef, + EmptyLayer: emptyLayer, } - if enableEarlyCommit { - options.LayerIndex = &layerIndex - } + options.LayerIndex = &layerIndex reused, blobInfo, err = dest.TryReusingBlobWithOptions(ctx, srcInfo, options) } else { reused, blobInfo, err = ic.c.dest.TryReusingBlob(ctx, srcInfo, ic.c.blobInfoCache, ic.canSubstituteBlobs) @@ -1245,7 +1253,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to bar := ic.c.createProgressBar(pool, srcInfo, "blob", "done") - blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar, layerIndex) + blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar, layerIndex, emptyLayer) if err != nil { return types.BlobInfo{}, "", err } @@ -1276,7 +1284,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // perhaps (de/re/)compressing the stream, // and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller. func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, - diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, <-chan diffIDResult, error) { + diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar, layerIndex int, emptyLayer bool) (types.BlobInfo, <-chan diffIDResult, error) { var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil var diffIDChan chan diffIDResult @@ -1301,7 +1309,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea } } - blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar, layerIndex) // Sets err to nil on success + blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar, layerIndex, emptyLayer) // Sets err to nil on success return blobInfo, diffIDChan, err // We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan } @@ -1353,7 +1361,7 @@ func (r errorAnnotationReader) Read(b []byte) (n int, err error) { // and returns a complete blobInfo of the copied blob. func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer, - canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, error) { + canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar, layerIndex int, emptyLayer bool) (types.BlobInfo, error) { if isConfig { // This is guaranteed by the caller, but set it here to be explicit. canModifyBlob = false } @@ -1556,10 +1564,11 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions) if ok { options := internalTypes.PutBlobOptions{ - Cache: c.blobInfoCache, - IsConfig: isConfig, + Cache: c.blobInfoCache, + IsConfig: isConfig, + EmptyLayer: emptyLayer, } - if !isConfig && enableEarlyCommit { + if !isConfig { options.LayerIndex = &layerIndex } uploadedInfo, err = dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options) diff --git a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go index 3e81e06c0..3bda6af29 100644 --- a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go +++ b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go @@ -9,7 +9,7 @@ package platform // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/vendor/github.com/containers/image/v5/internal/types/types.go b/vendor/github.com/containers/image/v5/internal/types/types.go index bf89a69b8..4a863ba34 100644 --- a/vendor/github.com/containers/image/v5/internal/types/types.go +++ b/vendor/github.com/containers/image/v5/internal/types/types.go @@ -39,6 +39,8 @@ type PutBlobOptions struct { Cache publicTypes.BlobInfoCache // Denotes whether the blob is a config or not. IsConfig bool + // Indicates an empty layer. + EmptyLayer bool // The corresponding index in the layer slice. LayerIndex *int } @@ -49,6 +51,8 @@ type TryReusingBlobOptions struct { Cache publicTypes.BlobInfoCache // Use an equivalent of the desired blob. CanSubstitute bool + // Indicates an empty layer. + EmptyLayer bool // The corresponding index in the layer slice. LayerIndex *int // The reference of the image that contains the target blob. diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go index b84aac6e4..ec7c2fcc3 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go @@ -51,21 +51,26 @@ var ( ErrNotSupported = errors.New("not supported") ) -// SetAuthentication stores the username and password in the credential helper or file -func SetAuthentication(sys *types.SystemContext, registry, username, password string) error { +// SetCredentials stores the username and password in the credential helper or file +// and returns path to file or helper name in format (helper:%s). +// Returns a human-redable description of the location that was updated. +// NOTE: The return value is only intended to be read by humans; its form is not an API, +// it may change (or new forms can be added) any time. +func SetCredentials(sys *types.SystemContext, registry, username, password string) (string, error) { helpers, err := sysregistriesv2.CredentialHelpers(sys) if err != nil { - return err + return "", err } // Make sure to collect all errors. var multiErr error for _, helper := range helpers { + var desc string var err error switch helper { // Special-case the built-in helpers for auth files. case sysregistriesv2.AuthenticationFileHelper: - err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + desc, err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { if ch, exists := auths.CredHelpers[registry]; exists { return false, setAuthToCredHelper(ch, registry, username, password) } @@ -76,6 +81,7 @@ func SetAuthentication(sys *types.SystemContext, registry, username, password st }) // External helpers. default: + desc = fmt.Sprintf("credential helper: %s", helper) err = setAuthToCredHelper(helper, registry, username, password) } if err != nil { @@ -84,9 +90,15 @@ func SetAuthentication(sys *types.SystemContext, registry, username, password st continue } logrus.Debugf("Stored credentials for %s in credential helper %s", registry, helper) - return nil + return desc, nil } - return multiErr + return "", multiErr +} + +// SetAuthentication stores the username and password in the credential helper or file +func SetAuthentication(sys *types.SystemContext, registry, username, password string) error { + _, err := SetCredentials(sys, registry, username, password) + return err } // GetAllCredentials returns the registry credentials for all registries stored @@ -322,7 +334,7 @@ func RemoveAuthentication(sys *types.SystemContext, registry string) error { switch helper { // Special-case the built-in helper for auth files. case sysregistriesv2.AuthenticationFileHelper: - err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + _, err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { if innerHelper, exists := auths.CredHelpers[registry]; exists { removeFromCredHelper(innerHelper) } @@ -368,7 +380,7 @@ func RemoveAllAuthentication(sys *types.SystemContext) error { switch helper { // Special-case the built-in helper for auth files. case sysregistriesv2.AuthenticationFileHelper: - err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + _, err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { for registry, helper := range auths.CredHelpers { // Helpers in auth files are expected // to exist, so no special treatment @@ -493,42 +505,44 @@ func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) { return auths, nil } -// modifyJSON writes to auth.json if the dockerConfigFile has been updated -func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) error { +// modifyJSON finds an auth.json file, calls editor on the contents, and +// writes it back if editor returns true. +// Returns a human-redable description of the file, to be returned by SetCredentials. +func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (bool, error)) (string, error) { path, legacyFormat, err := getPathToAuth(sys) if err != nil { - return err + return "", err } if legacyFormat { - return fmt.Errorf("writes to %s using legacy format are not supported", path) + return "", fmt.Errorf("writes to %s using legacy format are not supported", path) } dir := filepath.Dir(path) if err = os.MkdirAll(dir, 0700); err != nil { - return err + return "", err } auths, err := readJSONFile(path, false) if err != nil { - return errors.Wrapf(err, "error reading JSON file %q", path) + return "", errors.Wrapf(err, "error reading JSON file %q", path) } updated, err := editor(&auths) if err != nil { - return errors.Wrapf(err, "error updating %q", path) + return "", errors.Wrapf(err, "error updating %q", path) } if updated { newData, err := json.MarshalIndent(auths, "", "\t") if err != nil { - return errors.Wrapf(err, "error marshaling JSON %q", path) + return "", errors.Wrapf(err, "error marshaling JSON %q", path) } if err = ioutil.WriteFile(path, newData, 0600); err != nil { - return errors.Wrapf(err, "error writing to file %q", path) + return "", errors.Wrapf(err, "error writing to file %q", path) } } - return nil + return path, nil } func getAuthFromCredHelper(credHelper, registry string) (types.DockerAuthConfig, error) { diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go index 5bbfb450f..93c75b944 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go @@ -13,9 +13,9 @@ import ( // reenable keyring support, we should introduce a similar built-in credential // helpers as for `sysregistriesv2.AuthenticationFileHelper`. -const keyDescribePrefix = "container-registry-login:" // nolint +const keyDescribePrefix = "container-registry-login:" //nolint:deadcode,unused -func getAuthFromKernelKeyring(registry string) (string, string, error) { // nolint +func getAuthFromKernelKeyring(registry string) (string, string, error) { //nolint:deadcode,unused userkeyring, err := keyctl.UserKeyring() if err != nil { return "", "", err @@ -35,7 +35,7 @@ func getAuthFromKernelKeyring(registry string) (string, string, error) { // noli return parts[0], parts[1], nil } -func deleteAuthFromKernelKeyring(registry string) error { // nolint +func deleteAuthFromKernelKeyring(registry string) error { //nolint:deadcode,unused userkeyring, err := keyctl.UserKeyring() if err != nil { @@ -48,7 +48,7 @@ func deleteAuthFromKernelKeyring(registry string) error { // nolint return key.Unlink() } -func removeAllAuthFromKernelKeyring() error { // nolint +func removeAllAuthFromKernelKeyring() error { //nolint:deadcode,unused keys, err := keyctl.ReadUserKeyring() if err != nil { return err @@ -81,7 +81,7 @@ func removeAllAuthFromKernelKeyring() error { // nolint return nil } -func setAuthToKernelKeyring(registry, username, password string) error { // nolint +func setAuthToKernelKeyring(registry, username, password string) error { //nolint:deadcode,unused keyring, err := keyctl.SessionKeyring() if err != nil { return err @@ -114,6 +114,6 @@ func setAuthToKernelKeyring(registry, username, password string) error { // noli return nil } -func genDescription(registry string) string { // nolint +func genDescription(registry string) string { //nolint:deadcode,unused return fmt.Sprintf("%s%s", keyDescribePrefix, registry) } diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config_unsupported.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config_unsupported.go index 9b0e8bee2..65e580410 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config_unsupported.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config_unsupported.go @@ -3,18 +3,18 @@ package config -func getAuthFromKernelKeyring(registry string) (string, string, error) { +func getAuthFromKernelKeyring(registry string) (string, string, error) { //nolint:deadcode,unused return "", "", ErrNotSupported } -func deleteAuthFromKernelKeyring(registry string) error { +func deleteAuthFromKernelKeyring(registry string) error { //nolint:deadcode,unused return ErrNotSupported } -func setAuthToKernelKeyring(registry, username, password string) error { +func setAuthToKernelKeyring(registry, username, password string) error { //nolint:deadcode,unused return ErrNotSupported } -func removeAllAuthFromKernelKeyring() error { +func removeAllAuthFromKernelKeyring() error { //nolint:deadcode,unused return ErrNotSupported } diff --git a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go index b64f44674..ab1eee8f3 100644 --- a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go +++ b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go @@ -11,7 +11,7 @@ import ( "github.com/manifoldco/promptui" "github.com/opencontainers/go-digest" "github.com/pkg/errors" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // IsShortName returns true if the specified input is a "short name". A "short @@ -343,7 +343,7 @@ func Resolve(ctx *types.SystemContext, name string) (*Resolved, error) { } // If we don't have a TTY, act according to the mode. - if !terminal.IsTerminal(int(os.Stdout.Fd())) || !terminal.IsTerminal(int(os.Stdin.Fd())) { + if !term.IsTerminal(int(os.Stdout.Fd())) || !term.IsTerminal(int(os.Stdin.Fd())) { switch mode { case types.ShortNameModePermissive: // Permissive falls back to using all candidates. diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go index f4747357c..7072d6860 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_image.go +++ b/vendor/github.com/containers/image/v5/storage/storage_image.go @@ -76,12 +76,12 @@ type storageImageDestination struct { indexToStorageID map[int]*string // All accesses to below data are protected by `lock` which is made // *explicit* in the code. - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) - indexToPulledBlob map[int]*types.BlobInfo // Mapping from layer (by index) to pulled down blob - blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) + indexToPulledLayerInfo map[int]*manifest.LayerInfo // Mapping from layer (by index) to pulled down blob + blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer } type storageImageCloser struct { @@ -392,17 +392,17 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* return nil, errors.Wrapf(err, "error creating a temporary directory") } image := &storageImageDestination{ - imageRef: imageRef, - directory: directory, - signatureses: make(map[digest.Digest][]byte), - blobDiffIDs: make(map[digest.Digest]digest.Digest), - blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, - SignaturesSizes: make(map[digest.Digest][]int), - indexToStorageID: make(map[int]*string), - indexToPulledBlob: make(map[int]*types.BlobInfo), + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + blobDiffIDs: make(map[digest.Digest]digest.Digest), + blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), + indexToStorageID: make(map[int]*string), + indexToPulledLayerInfo: make(map[int]*manifest.LayerInfo), } return image, nil } @@ -449,7 +449,7 @@ func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream return info, nil } - return info, s.queueOrCommit(ctx, info, *options.LayerIndex) + return info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) } // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. @@ -542,7 +542,7 @@ func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, return reused, info, err } - return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex) + return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) } // tryReusingBlobWithSrcRef is a wrapper around TryReusingBlob. @@ -731,7 +731,7 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er // queueOrCommit queues in the specified blob to be committed to the storage. // If no other goroutine is already committing layers, the layer and all // subsequent layers (if already queued) will be committed to the storage. -func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int) error { +func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int, emptyLayer bool) error { // NOTE: whenever the code below is touched, make sure that all code // paths unlock the lock and to unlock it exactly once. // @@ -751,7 +751,10 @@ func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types. // caller is the "worker" routine comitting layers. All other routines // can continue pulling and queuing in layers. s.lock.Lock() - s.indexToPulledBlob[index] = &blob + s.indexToPulledLayerInfo[index] = &manifest.LayerInfo{ + BlobInfo: blob, + EmptyLayer: emptyLayer, + } // We're still waiting for at least one previous/parent layer to be // committed, so there's nothing to do. @@ -760,14 +763,10 @@ func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types. return nil } - for info := s.indexToPulledBlob[index]; info != nil; info = s.indexToPulledBlob[index] { + for info := s.indexToPulledLayerInfo[index]; info != nil; info = s.indexToPulledLayerInfo[index] { s.lock.Unlock() - layerInfo := manifest.LayerInfo{ - BlobInfo: *info, - EmptyLayer: info.Digest == image.GzippedEmptyLayerDigest, - } // Note: commitLayer locks on-demand. - if err := s.commitLayer(ctx, layerInfo, index); err != nil { + if err := s.commitLayer(ctx, *info, index); err != nil { return err } s.lock.Lock() @@ -1034,25 +1033,6 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t return errors.Wrapf(err, "error saving big data %q for image %q", blob.String(), img.ID) } } - // Set the reference's name on the image. We don't need to worry about avoiding duplicate - // values because SetNames() will deduplicate the list that we pass to it. - if name := s.imageRef.DockerReference(); len(oldNames) > 0 || name != nil { - names := []string{} - if name != nil { - names = append(names, name.String()) - } - if len(oldNames) > 0 { - names = append(names, oldNames...) - } - if err := s.imageRef.transport.store.SetNames(img.ID, names); err != nil { - if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { - logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) - } - logrus.Debugf("error setting names %v on image %q: %v", names, img.ID, err) - return errors.Wrapf(err, "error setting names %v on image %q", names, img.ID) - } - logrus.Debugf("set names of image %q to %v", img.ID, names) - } // Save the unparsedToplevel's manifest. if len(toplevelManifest) != 0 { manifestDigest, err := manifest.Digest(toplevelManifest) @@ -1130,6 +1110,25 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t } logrus.Debugf("saved image metadata %q", string(metadata)) } + // Set the reference's name on the image. We don't need to worry about avoiding duplicate + // values because SetNames() will deduplicate the list that we pass to it. + if name := s.imageRef.DockerReference(); len(oldNames) > 0 || name != nil { + names := []string{} + if name != nil { + names = append(names, name.String()) + } + if len(oldNames) > 0 { + names = append(names, oldNames...) + } + if err := s.imageRef.transport.store.SetNames(img.ID, names); err != nil { + if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil { + logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2) + } + logrus.Debugf("error setting names %v on image %q: %v", names, img.ID, err) + return errors.Wrapf(err, "error setting names %v on image %q", names, img.ID) + } + logrus.Debugf("set names of image %q to %v", img.ID, names) + } return nil } diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 4afb3b90b..edf4681de 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -6,9 +6,9 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 12 + VersionMinor = 13 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/docker/docker-credential-helpers/client/command.go b/vendor/github.com/docker/docker-credential-helpers/client/command.go index 8da334306..0183c0639 100644 --- a/vendor/github.com/docker/docker-credential-helpers/client/command.go +++ b/vendor/github.com/docker/docker-credential-helpers/client/command.go @@ -4,7 +4,8 @@ import ( "fmt" "io" "os" - "os/exec" + + exec "golang.org/x/sys/execabs" ) // Program is an interface to execute external programs. diff --git a/vendor/github.com/docker/docker-credential-helpers/credentials/version.go b/vendor/github.com/docker/docker-credential-helpers/credentials/version.go index c2cc3e2e0..185e36796 100644 --- a/vendor/github.com/docker/docker-credential-helpers/credentials/version.go +++ b/vendor/github.com/docker/docker-credential-helpers/credentials/version.go @@ -1,4 +1,4 @@ package credentials // Version holds a string describing the current version -const Version = "0.6.3" +const Version = "0.6.4" diff --git a/vendor/github.com/imdario/mergo/.travis.yml b/vendor/github.com/imdario/mergo/.travis.yml index dad29725f..d324c43ba 100644 --- a/vendor/github.com/imdario/mergo/.travis.yml +++ b/vendor/github.com/imdario/mergo/.travis.yml @@ -1,4 +1,7 @@ language: go +arch: + - amd64 + - ppc64le install: - go get -t - go get golang.org/x/tools/cmd/cover diff --git a/vendor/github.com/imdario/mergo/README.md b/vendor/github.com/imdario/mergo/README.md index 876abb500..aa8cbd7ce 100644 --- a/vendor/github.com/imdario/mergo/README.md +++ b/vendor/github.com/imdario/mergo/README.md @@ -97,7 +97,7 @@ If Mergo is useful to you, consider buying me a coffee, a beer, or making a mont - [mantasmatelis/whooplist-server](https://github.com/mantasmatelis/whooplist-server) - [jnuthong/item_search](https://github.com/jnuthong/item_search) - [bukalapak/snowboard](https://github.com/bukalapak/snowboard) -- [janoszen/containerssh](https://github.com/janoszen/containerssh) +- [containerssh/containerssh](https://github.com/containerssh/containerssh) ## Install diff --git a/vendor/github.com/imdario/mergo/merge.go b/vendor/github.com/imdario/mergo/merge.go index afa84a1e2..8c2a8fcd9 100644 --- a/vendor/github.com/imdario/mergo/merge.go +++ b/vendor/github.com/imdario/mergo/merge.go @@ -95,13 +95,18 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co } } } else { - if (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) { + if dst.CanSet() && (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) { dst.Set(src) } } case reflect.Map: if dst.IsNil() && !src.IsNil() { - dst.Set(reflect.MakeMap(dst.Type())) + if dst.CanSet() { + dst.Set(reflect.MakeMap(dst.Type())) + } else { + dst = src + return + } } if src.Kind() != reflect.Map { diff --git a/vendor/github.com/jinzhu/copier/README.md b/vendor/github.com/jinzhu/copier/README.md index cff72405c..ec04b4be0 100644 --- a/vendor/github.com/jinzhu/copier/README.md +++ b/vendor/github.com/jinzhu/copier/README.md @@ -27,9 +27,10 @@ import ( ) type User struct { - Name string - Role string - Age int32 + Name string + Role string + Age int32 + EmployeCode int64 `copier:"EmployeNum"` // specify field name // Explicitly ignored in the destination struct. Salary int @@ -52,7 +53,7 @@ type Employee struct { Salary int `copier:"-"` DoubleAge int32 - EmployeId int64 + EmployeId int64 `copier:"EmployeNum"` // specify field name SuperRole string } diff --git a/vendor/github.com/jinzhu/copier/copier.go b/vendor/github.com/jinzhu/copier/copier.go index 72bf65c78..412ff5497 100644 --- a/vendor/github.com/jinzhu/copier/copier.go +++ b/vendor/github.com/jinzhu/copier/copier.go @@ -3,9 +3,11 @@ package copier import ( "database/sql" "database/sql/driver" + "errors" "fmt" "reflect" "strings" + "unicode" ) // These flags define options for tag handling @@ -32,6 +34,19 @@ type Option struct { DeepCopy bool } +// Tag Flags +type flags struct { + BitFlags map[string]uint8 + SrcNames tagNameMapping + DestNames tagNameMapping +} + +// Field Tag name mapping +type tagNameMapping struct { + FieldNameToTag map[string]string + TagToFieldName map[string]string +} + // Copy copy things func Copy(toValue interface{}, fromValue interface{}) (err error) { return copier(toValue, fromValue, Option{}) @@ -134,7 +149,8 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) } if !set(to.Index(i), from.Index(i), opt.DeepCopy) { - err = CopyWithOption(to.Index(i).Addr().Interface(), from.Index(i).Interface(), opt) + // ignore error while copy slice element + err = copier(to.Index(i).Addr().Interface(), from.Index(i).Interface(), opt) if err != nil { continue } @@ -148,7 +164,7 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) return } - if to.Kind() == reflect.Slice { + if from.Kind() == reflect.Slice || to.Kind() == reflect.Slice { isSlice = true if from.Kind() == reflect.Slice { amount = from.Len() @@ -180,9 +196,9 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) } // Get tag options - tagBitFlags := map[string]uint8{} - if dest.IsValid() { - tagBitFlags = getBitFlags(toType) + flgs, err := getFlags(dest, source, toType, fromType) + if err != nil { + return err } // check source @@ -193,17 +209,18 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) name := field.Name // Get bit flags for field - fieldFlags, _ := tagBitFlags[name] + fieldFlags, _ := flgs.BitFlags[name] // Check if we should ignore copying if (fieldFlags & tagIgnore) != 0 { continue } - if fromField := source.FieldByName(name); fromField.IsValid() && !shouldIgnore(fromField, opt.IgnoreEmpty) { + srcFieldName, destFieldName := getFieldName(name, flgs) + if fromField := source.FieldByName(srcFieldName); fromField.IsValid() && !shouldIgnore(fromField, opt.IgnoreEmpty) { // process for nested anonymous field destFieldNotSet := false - if f, ok := dest.Type().FieldByName(name); ok { + if f, ok := dest.Type().FieldByName(destFieldName); ok { for idx := range f.Index { destField := dest.FieldByIndex(f.Index[:idx+1]) @@ -229,7 +246,7 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) break } - toField := dest.FieldByName(name) + toField := dest.FieldByName(destFieldName) if toField.IsValid() { if toField.CanSet() { if !set(toField, fromField, opt.DeepCopy) { @@ -239,16 +256,16 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) } if fieldFlags != 0 { // Note that a copy was made - tagBitFlags[name] = fieldFlags | hasCopied + flgs.BitFlags[name] = fieldFlags | hasCopied } } } else { // try to set to method var toMethod reflect.Value if dest.CanAddr() { - toMethod = dest.Addr().MethodByName(name) + toMethod = dest.Addr().MethodByName(destFieldName) } else { - toMethod = dest.MethodByName(name) + toMethod = dest.MethodByName(destFieldName) } if toMethod.IsValid() && toMethod.Type().NumIn() == 1 && fromField.Type().AssignableTo(toMethod.Type().In(0)) { @@ -261,16 +278,17 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) // Copy from from method to dest field for _, field := range deepFields(toType) { name := field.Name + srcFieldName, destFieldName := getFieldName(name, flgs) var fromMethod reflect.Value if source.CanAddr() { - fromMethod = source.Addr().MethodByName(name) + fromMethod = source.Addr().MethodByName(srcFieldName) } else { - fromMethod = source.MethodByName(name) + fromMethod = source.MethodByName(srcFieldName) } if fromMethod.IsValid() && fromMethod.Type().NumIn() == 0 && fromMethod.Type().NumOut() == 1 && !shouldIgnore(fromMethod, opt.IgnoreEmpty) { - if toField := dest.FieldByName(name); toField.IsValid() && toField.CanSet() { + if toField := dest.FieldByName(destFieldName); toField.IsValid() && toField.CanSet() { values := fromMethod.Call([]reflect.Value{}) if len(values) >= 1 { set(toField, values[0], opt.DeepCopy) @@ -280,25 +298,37 @@ func copier(toValue interface{}, fromValue interface{}, opt Option) (err error) } } - if isSlice { + if isSlice && to.Kind() == reflect.Slice { if dest.Addr().Type().AssignableTo(to.Type().Elem()) { if to.Len() < i+1 { to.Set(reflect.Append(to, dest.Addr())) } else { - set(to.Index(i), dest.Addr(), opt.DeepCopy) + if !set(to.Index(i), dest.Addr(), opt.DeepCopy) { + // ignore error while copy slice element + err = copier(to.Index(i).Addr().Interface(), dest.Addr().Interface(), opt) + if err != nil { + continue + } + } } } else if dest.Type().AssignableTo(to.Type().Elem()) { if to.Len() < i+1 { to.Set(reflect.Append(to, dest)) } else { - set(to.Index(i), dest, opt.DeepCopy) + if !set(to.Index(i), dest, opt.DeepCopy) { + // ignore error while copy slice element + err = copier(to.Index(i).Addr().Interface(), dest.Interface(), opt) + if err != nil { + continue + } + } } } } else if initDest { to.Set(dest) } - err = checkBitFlags(tagBitFlags) + err = checkBitFlags(flgs.BitFlags) } return @@ -432,46 +462,90 @@ func set(to, from reflect.Value, deepCopy bool) bool { } // parseTags Parses struct tags and returns uint8 bit flags. -func parseTags(tag string) (flags uint8) { +func parseTags(tag string) (flg uint8, name string, err error) { for _, t := range strings.Split(tag, ",") { switch t { case "-": - flags = tagIgnore + flg = tagIgnore return case "must": - flags = flags | tagMust + flg = flg | tagMust case "nopanic": - flags = flags | tagNoPanic + flg = flg | tagNoPanic + default: + if unicode.IsUpper([]rune(t)[0]) { + name = strings.TrimSpace(t) + } else { + err = errors.New("copier field name tag must be start upper case") + } } } return } -// getBitFlags Parses struct tags for bit flags. -func getBitFlags(toType reflect.Type) map[string]uint8 { - flags := map[string]uint8{} - toTypeFields := deepFields(toType) +// getTagFlags Parses struct tags for bit flags, field name. +func getFlags(dest, src reflect.Value, toType, fromType reflect.Type) (flags, error) { + flgs := flags{ + BitFlags: map[string]uint8{}, + SrcNames: tagNameMapping{ + FieldNameToTag: map[string]string{}, + TagToFieldName: map[string]string{}, + }, + DestNames: tagNameMapping{ + FieldNameToTag: map[string]string{}, + TagToFieldName: map[string]string{}, + }, + } + var toTypeFields, fromTypeFields []reflect.StructField + if dest.IsValid() { + toTypeFields = deepFields(toType) + } + if src.IsValid() { + fromTypeFields = deepFields(fromType) + } // Get a list dest of tags for _, field := range toTypeFields { tags := field.Tag.Get("copier") if tags != "" { - flags[field.Name] = parseTags(tags) + var name string + var err error + if flgs.BitFlags[field.Name], name, err = parseTags(tags); err != nil { + return flags{}, err + } else if name != "" { + flgs.DestNames.FieldNameToTag[field.Name] = name + flgs.DestNames.TagToFieldName[name] = field.Name + } + } + } + + // Get a list source of tags + for _, field := range fromTypeFields { + tags := field.Tag.Get("copier") + if tags != "" { + var name string + var err error + if _, name, err = parseTags(tags); err != nil { + return flags{}, err + } else if name != "" { + flgs.SrcNames.FieldNameToTag[field.Name] = name + flgs.SrcNames.TagToFieldName[name] = field.Name + } } } - return flags + return flgs, nil } // checkBitFlags Checks flags for error or panic conditions. func checkBitFlags(flagsList map[string]uint8) (err error) { // Check flag conditions were met - for name, flags := range flagsList { - if flags&hasCopied == 0 { + for name, flgs := range flagsList { + if flgs&hasCopied == 0 { switch { - case flags&tagMust != 0 && flags&tagNoPanic != 0: + case flgs&tagMust != 0 && flgs&tagNoPanic != 0: err = fmt.Errorf("field %s has must tag but was not copied", name) return - case flags&(tagMust) != 0: + case flgs&(tagMust) != 0: panic(fmt.Sprintf("Field %s has must tag but was not copied", name)) } } @@ -479,6 +553,40 @@ func checkBitFlags(flagsList map[string]uint8) (err error) { return } +func getFieldName(fieldName string, flgs flags) (srcFieldName string, destFieldName string) { + // get dest field name + if srcTagName, ok := flgs.SrcNames.FieldNameToTag[fieldName]; ok { + destFieldName = srcTagName + if destTagName, ok := flgs.DestNames.TagToFieldName[srcTagName]; ok { + destFieldName = destTagName + } + } else { + if destTagName, ok := flgs.DestNames.TagToFieldName[fieldName]; ok { + destFieldName = destTagName + } + } + if destFieldName == "" { + destFieldName = fieldName + } + + // get source field name + if destTagName, ok := flgs.DestNames.FieldNameToTag[fieldName]; ok { + srcFieldName = destTagName + if srcField, ok := flgs.SrcNames.TagToFieldName[destTagName]; ok { + srcFieldName = srcField + } + } else { + if srcField, ok := flgs.SrcNames.TagToFieldName[fieldName]; ok { + srcFieldName = srcField + } + } + + if srcFieldName == "" { + srcFieldName = fieldName + } + return +} + func driverValuer(v reflect.Value) (i driver.Valuer, ok bool) { if !v.CanAddr() { diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index e30af505c..8a98c4562 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -168,10 +168,10 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { // Read block data. if cap(b.dataStorage) < cSize { - if b.lowMem { + if b.lowMem || cSize > maxCompressedBlockSize { b.dataStorage = make([]byte, 0, cSize) } else { - b.dataStorage = make([]byte, 0, maxBlockSize) + b.dataStorage = make([]byte, 0, maxCompressedBlockSize) } } if cap(b.dst) <= maxSize { diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index c0fd058c2..95cc9b8b8 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -17,14 +17,16 @@ type decoderOptions struct { lowMem bool concurrent int maxDecodedSize uint64 + maxWindowSize uint64 dicts []dict } func (o *decoderOptions) setDefault() { *o = decoderOptions{ // use less ram: true for now, but may change. - lowMem: true, - concurrent: runtime.GOMAXPROCS(0), + lowMem: true, + concurrent: runtime.GOMAXPROCS(0), + maxWindowSize: MaxWindowSize, } o.maxDecodedSize = 1 << 63 } @@ -52,7 +54,6 @@ func WithDecoderConcurrency(n int) DOption { // WithDecoderMaxMemory allows to set a maximum decoded size for in-memory // non-streaming operations or maximum window size for streaming operations. // This can be used to control memory usage of potentially hostile content. -// For streaming operations, the maximum window size is capped at 1<<30 bytes. // Maximum and default is 1 << 63 bytes. func WithDecoderMaxMemory(n uint64) DOption { return func(o *decoderOptions) error { @@ -81,3 +82,21 @@ func WithDecoderDicts(dicts ...[]byte) DOption { return nil } } + +// WithDecoderMaxWindow allows to set a maximum window size for decodes. +// This allows rejecting packets that will cause big memory usage. +// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. +// If WithDecoderMaxMemory is set to a lower value, that will be used. +// Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. +func WithDecoderMaxWindow(size uint64) DOption { + return func(o *decoderOptions) error { + if size < MinWindowSize { + return errors.New("WithMaxWindowSize must be at least 1KB, 1024 bytes") + } + if size > (1<<41)+7*(1<<38) { + return errors.New("WithMaxWindowSize must be less than (1<<41) + 7*(1<<38) ~ 3.75TB") + } + o.maxWindowSize = size + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index e8cc9a2c2..989c79f8c 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -22,10 +22,6 @@ type frameDec struct { WindowSize uint64 - // maxWindowSize is the maximum windows size to support. - // should never be bigger than max-int. - maxWindowSize uint64 - // In order queue of blocks being decoded. decoding chan *blockDec @@ -50,8 +46,11 @@ type frameDec struct { } const ( - // The minimum Window_Size is 1 KB. + // MinWindowSize is the minimum Window Size, which is 1 KB. MinWindowSize = 1 << 10 + + // MaxWindowSize is the maximum encoder window size + // and the default decoder maximum window size. MaxWindowSize = 1 << 29 ) @@ -61,12 +60,11 @@ var ( ) func newFrameDec(o decoderOptions) *frameDec { - d := frameDec{ - o: o, - maxWindowSize: MaxWindowSize, + if o.maxWindowSize > o.maxDecodedSize { + o.maxWindowSize = o.maxDecodedSize } - if d.maxWindowSize > o.maxDecodedSize { - d.maxWindowSize = o.maxDecodedSize + d := frameDec{ + o: o, } return &d } @@ -251,13 +249,17 @@ func (d *frameDec) reset(br byteBuffer) error { } } - if d.WindowSize > d.maxWindowSize { - printf("window size %d > max %d\n", d.WindowSize, d.maxWindowSize) + if d.WindowSize > uint64(d.o.maxWindowSize) { + if debugDecoder { + printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) + } return ErrWindowSizeExceeded } // The minimum Window_Size is 1 KB. if d.WindowSize < MinWindowSize { - println("got window size: ", d.WindowSize) + if debugDecoder { + println("got window size: ", d.WindowSize) + } return ErrWindowSizeTooSmall } d.history.windowSize = int(d.WindowSize) @@ -352,8 +354,8 @@ func (d *frameDec) checkCRC() error { func (d *frameDec) initAsync() { if !d.o.lowMem && !d.SingleSegment { - // set max extra size history to 10MB. - d.history.maxSize = d.history.windowSize + maxBlockSize*5 + // set max extra size history to 2MB. + d.history.maxSize = d.history.windowSize + maxBlockSize } // re-alloc if more than one extra block size. if d.o.lowMem && cap(d.history.b) > d.history.maxSize+maxBlockSize { diff --git a/vendor/github.com/mattn/go-runewidth/go.mod b/vendor/github.com/mattn/go-runewidth/go.mod index 8a9d524ec..62dba1bfc 100644 --- a/vendor/github.com/mattn/go-runewidth/go.mod +++ b/vendor/github.com/mattn/go-runewidth/go.mod @@ -2,4 +2,4 @@ module github.com/mattn/go-runewidth go 1.9 -require github.com/rivo/uniseg v0.1.0 +require github.com/rivo/uniseg v0.2.0 diff --git a/vendor/github.com/mattn/go-runewidth/go.sum b/vendor/github.com/mattn/go-runewidth/go.sum index 02135660b..03f902d56 100644 --- a/vendor/github.com/mattn/go-runewidth/go.sum +++ b/vendor/github.com/mattn/go-runewidth/go.sum @@ -1,2 +1,2 @@ -github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/vendor/github.com/vbauerster/mpb/v7/.gitignore b/vendor/github.com/vbauerster/mpb/v7/.gitignore new file mode 100644 index 000000000..63bd91672 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/.gitignore @@ -0,0 +1,5 @@ +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out diff --git a/vendor/github.com/vbauerster/mpb/v7/.travis.yml b/vendor/github.com/vbauerster/mpb/v7/.travis.yml new file mode 100644 index 000000000..9a203a67d --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/.travis.yml @@ -0,0 +1,11 @@ +language: go +arch: + - amd64 + - ppc64le + +go: + - 1.14.x + +script: + - go test -race ./... + - for i in _examples/*/; do go build $i/*.go || exit 1; done diff --git a/vendor/github.com/vbauerster/mpb/v7/README.md b/vendor/github.com/vbauerster/mpb/v7/README.md new file mode 100644 index 000000000..d0560d799 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/README.md @@ -0,0 +1,121 @@ +# Multi Progress Bar + +[](https://pkg.go.dev/github.com/vbauerster/mpb/v7) +[](https://travis-ci.org/vbauerster/mpb) +[](https://goreportcard.com/report/github.com/vbauerster/mpb) +[](https://www.paypal.me/vbauerster) + +**mpb** is a Go lib for rendering progress bars in terminal applications. + +## Features + +- **Multiple Bars**: Multiple progress bars are supported +- **Dynamic Total**: Set total while bar is running +- **Dynamic Add/Remove**: Dynamically add or remove bars +- **Cancellation**: Cancel whole rendering process +- **Predefined Decorators**: Elapsed time, [ewma](https://github.com/VividCortex/ewma) based ETA, Percentage, Bytes counter +- **Decorator's width sync**: Synchronized decorator's width among multiple bars + +## Usage + +#### [Rendering single bar](_examples/singleBar/main.go) + +```go +package main + +import ( + "math/rand" + "time" + + "github.com/vbauerster/mpb/v7" + "github.com/vbauerster/mpb/v7/decor" +) + +func main() { + // initialize progress container, with custom width + p := mpb.New(mpb.WithWidth(64)) + + total := 100 + name := "Single Bar:" + // adding a single bar, which will inherit container's width + bar := p.Add(int64(total), + // progress bar filler with customized style + mpb.NewBarFiller(mpb.BarStyle().Lbound("╢").Filler("▌").Tip("▌").Padding("░").Rbound("╟")), + mpb.PrependDecorators( + // display our name with one space on the right + decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), + // replace ETA decorator with "done" message, OnComplete event + decor.OnComplete( + decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done", + ), + ), + mpb.AppendDecorators(decor.Percentage()), + ) + // simulating some work + max := 100 * time.Millisecond + for i := 0; i < total; i++ { + time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10) + bar.Increment() + } + // wait for our bar to complete and flush + p.Wait() +} +``` + +#### [Rendering multiple bars](_examples/multiBars/main.go) + +```go + var wg sync.WaitGroup + // passed &wg will be accounted at p.Wait() call + p := mpb.New(mpb.WithWaitGroup(&wg)) + total, numBars := 100, 3 + wg.Add(numBars) + + for i := 0; i < numBars; i++ { + name := fmt.Sprintf("Bar#%d:", i) + bar := p.AddBar(int64(total), + mpb.PrependDecorators( + // simple name decorator + decor.Name(name), + // decor.DSyncWidth bit enables column width synchronization + decor.Percentage(decor.WCSyncSpace), + ), + mpb.AppendDecorators( + // replace ETA decorator with "done" message, OnComplete event + decor.OnComplete( + // ETA decorator with ewma age of 60 + decor.EwmaETA(decor.ET_STYLE_GO, 60), "done", + ), + ), + ) + // simulating some work + go func() { + defer wg.Done() + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + max := 100 * time.Millisecond + for i := 0; i < total; i++ { + // start variable is solely for EWMA calculation + // EWMA's unit of measure is an iteration's duration + start := time.Now() + time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10) + bar.Increment() + // we need to call DecoratorEwmaUpdate to fulfill ewma decorator's contract + bar.DecoratorEwmaUpdate(time.Since(start)) + } + }() + } + // Waiting for passed &wg and for all bars to complete and flush + p.Wait() +``` + +#### [Dynamic total](_examples/dynTotal/main.go) + + + +#### [Complex example](_examples/complex/main.go) + + + +#### [Bytes counters](_examples/io/main.go) + + diff --git a/vendor/github.com/vbauerster/mpb/v7/UNLICENSE b/vendor/github.com/vbauerster/mpb/v7/UNLICENSE new file mode 100644 index 000000000..68a49daad --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to <http://unlicense.org/> diff --git a/vendor/github.com/vbauerster/mpb/v7/bar.go b/vendor/github.com/vbauerster/mpb/v7/bar.go new file mode 100644 index 000000000..ed6c73eda --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/bar.go @@ -0,0 +1,492 @@ +package mpb + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "runtime/debug" + "strings" + "time" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" + "github.com/vbauerster/mpb/v7/decor" +) + +// Bar represents a progress bar. +type Bar struct { + priority int // used by heap + index int // used by heap + + extendedLines int + toShutdown bool + toDrop bool + noPop bool + hasEwmaDecorators bool + operateState chan func(*bState) + frameCh chan io.Reader + syncTableCh chan [][]chan int + completed chan bool + + // cancel is called either by user or on complete event + cancel func() + // done is closed after cacheState is assigned + done chan struct{} + // cacheState is populated, right after close(shutdown) + cacheState *bState + + container *Progress + dlogger *log.Logger + recoveredPanic interface{} +} + +type extenderFunc func(in io.Reader, reqWidth int, st decor.Statistics) (out io.Reader, lines int) + +// bState is actual bar state. It gets passed to *Bar.serve(...) monitor +// goroutine. +type bState struct { + id int + priority int + reqWidth int + total int64 + current int64 + refill int64 + lastN int64 + iterated bool + trimSpace bool + completed bool + completeFlushed bool + triggerComplete bool + dropOnComplete bool + noPop bool + aDecorators []decor.Decorator + pDecorators []decor.Decorator + averageDecorators []decor.AverageDecorator + ewmaDecorators []decor.EwmaDecorator + shutdownListeners []decor.ShutdownListener + bufP, bufB, bufA *bytes.Buffer + filler BarFiller + middleware func(BarFiller) BarFiller + extender extenderFunc + + // runningBar is a key for *pState.parkedBars + runningBar *Bar + + debugOut io.Writer +} + +func newBar(container *Progress, bs *bState) *Bar { + logPrefix := fmt.Sprintf("%sbar#%02d ", container.dlogger.Prefix(), bs.id) + ctx, cancel := context.WithCancel(container.ctx) + + bar := &Bar{ + container: container, + priority: bs.priority, + toDrop: bs.dropOnComplete, + noPop: bs.noPop, + operateState: make(chan func(*bState)), + frameCh: make(chan io.Reader, 1), + syncTableCh: make(chan [][]chan int, 1), + completed: make(chan bool, 1), + done: make(chan struct{}), + cancel: cancel, + dlogger: log.New(bs.debugOut, logPrefix, log.Lshortfile), + } + + go bar.serve(ctx, bs) + return bar +} + +// ProxyReader wraps r with metrics required for progress tracking. +// Panics if r is nil. +func (b *Bar) ProxyReader(r io.Reader) io.ReadCloser { + if r == nil { + panic("expected non nil io.Reader") + } + return newProxyReader(r, b) +} + +// ID returs id of the bar. +func (b *Bar) ID() int { + result := make(chan int) + select { + case b.operateState <- func(s *bState) { result <- s.id }: + return <-result + case <-b.done: + return b.cacheState.id + } +} + +// Current returns bar's current number, in other words sum of all increments. +func (b *Bar) Current() int64 { + result := make(chan int64) + select { + case b.operateState <- func(s *bState) { result <- s.current }: + return <-result + case <-b.done: + return b.cacheState.current + } +} + +// SetRefill sets refill flag with specified amount. +// The underlying BarFiller will change its visual representation, to +// indicate refill event. Refill event may be referred to some retry +// operation for example. +func (b *Bar) SetRefill(amount int64) { + select { + case b.operateState <- func(s *bState) { + s.refill = amount + }: + case <-b.done: + } +} + +// TraverseDecorators traverses all available decorators and calls cb func on each. +func (b *Bar) TraverseDecorators(cb func(decor.Decorator)) { + select { + case b.operateState <- func(s *bState) { + for _, decorators := range [...][]decor.Decorator{ + s.pDecorators, + s.aDecorators, + } { + for _, d := range decorators { + cb(extractBaseDecorator(d)) + } + } + }: + case <-b.done: + } +} + +// SetTotal sets total dynamically. +// If total is less than or equal to zero it takes progress' current value. +func (b *Bar) SetTotal(total int64, triggerComplete bool) { + select { + case b.operateState <- func(s *bState) { + s.triggerComplete = triggerComplete + if total <= 0 { + s.total = s.current + } else { + s.total = total + } + if s.triggerComplete && !s.completed { + s.current = s.total + s.completed = true + go b.refreshTillShutdown() + } + }: + case <-b.done: + } +} + +// SetCurrent sets progress' current to an arbitrary value. +// Setting a negative value will cause a panic. +func (b *Bar) SetCurrent(current int64) { + select { + case b.operateState <- func(s *bState) { + s.iterated = true + s.lastN = current - s.current + s.current = current + if s.triggerComplete && s.current >= s.total { + s.current = s.total + s.completed = true + go b.refreshTillShutdown() + } + }: + case <-b.done: + } +} + +// Increment is a shorthand for b.IncrInt64(1). +func (b *Bar) Increment() { + b.IncrInt64(1) +} + +// IncrBy is a shorthand for b.IncrInt64(int64(n)). +func (b *Bar) IncrBy(n int) { + b.IncrInt64(int64(n)) +} + +// IncrInt64 increments progress by amount of n. +func (b *Bar) IncrInt64(n int64) { + select { + case b.operateState <- func(s *bState) { + s.iterated = true + s.lastN = n + s.current += n + if s.triggerComplete && s.current >= s.total { + s.current = s.total + s.completed = true + go b.refreshTillShutdown() + } + }: + case <-b.done: + } +} + +// DecoratorEwmaUpdate updates all EWMA based decorators. Should be +// called on each iteration, because EWMA's unit of measure is an +// iteration's duration. Panics if called before *Bar.Incr... family +// methods. +func (b *Bar) DecoratorEwmaUpdate(dur time.Duration) { + select { + case b.operateState <- func(s *bState) { + ewmaIterationUpdate(false, s, dur) + }: + case <-b.done: + ewmaIterationUpdate(true, b.cacheState, dur) + } +} + +// DecoratorAverageAdjust adjusts all average based decorators. Call +// if you need to adjust start time of all average based decorators +// or after progress resume. +func (b *Bar) DecoratorAverageAdjust(start time.Time) { + select { + case b.operateState <- func(s *bState) { + for _, d := range s.averageDecorators { + d.AverageAdjust(start) + } + }: + case <-b.done: + } +} + +// SetPriority changes bar's order among multiple bars. Zero is highest +// priority, i.e. bar will be on top. If you don't need to set priority +// dynamically, better use BarPriority option. +func (b *Bar) SetPriority(priority int) { + select { + case <-b.done: + default: + b.container.setBarPriority(b, priority) + } +} + +// Abort interrupts bar's running goroutine. Call this, if you'd like +// to stop/remove bar before completion event. It has no effect after +// completion event. If drop is true bar will be removed as well. +func (b *Bar) Abort(drop bool) { + select { + case <-b.done: + default: + if drop { + b.container.dropBar(b) + } + b.cancel() + } +} + +// Completed reports whether the bar is in completed state. +func (b *Bar) Completed() bool { + select { + case b.operateState <- func(s *bState) { b.completed <- s.completed }: + return <-b.completed + case <-b.done: + return true + } +} + +func (b *Bar) serve(ctx context.Context, s *bState) { + defer b.container.bwg.Done() + for { + select { + case op := <-b.operateState: + op(s) + case <-ctx.Done(): + b.cacheState = s + close(b.done) + // Notifying decorators about shutdown event + for _, sl := range s.shutdownListeners { + sl.Shutdown() + } + return + } + } +} + +func (b *Bar) render(tw int) { + select { + case b.operateState <- func(s *bState) { + stat := newStatistics(tw, s) + defer func() { + // recovering if user defined decorator panics for example + if p := recover(); p != nil { + if b.recoveredPanic == nil { + s.extender = makePanicExtender(p) + b.toShutdown = !b.toShutdown + b.recoveredPanic = p + } + frame, lines := s.extender(nil, s.reqWidth, stat) + b.extendedLines = lines + b.frameCh <- frame + b.dlogger.Println(p) + } + s.completeFlushed = s.completed + }() + frame, lines := s.extender(s.draw(stat), s.reqWidth, stat) + b.extendedLines = lines + b.toShutdown = s.completed && !s.completeFlushed + b.frameCh <- frame + }: + case <-b.done: + s := b.cacheState + stat := newStatistics(tw, s) + var r io.Reader + if b.recoveredPanic == nil { + r = s.draw(stat) + } + frame, lines := s.extender(r, s.reqWidth, stat) + b.extendedLines = lines + b.frameCh <- frame + } +} + +func (b *Bar) subscribeDecorators() { + var averageDecorators []decor.AverageDecorator + var ewmaDecorators []decor.EwmaDecorator + var shutdownListeners []decor.ShutdownListener + b.TraverseDecorators(func(d decor.Decorator) { + if d, ok := d.(decor.AverageDecorator); ok { + averageDecorators = append(averageDecorators, d) + } + if d, ok := d.(decor.EwmaDecorator); ok { + ewmaDecorators = append(ewmaDecorators, d) + } + if d, ok := d.(decor.ShutdownListener); ok { + shutdownListeners = append(shutdownListeners, d) + } + }) + select { + case b.operateState <- func(s *bState) { + s.averageDecorators = averageDecorators + s.ewmaDecorators = ewmaDecorators + s.shutdownListeners = shutdownListeners + }: + b.hasEwmaDecorators = len(ewmaDecorators) != 0 + case <-b.done: + } +} + +func (b *Bar) refreshTillShutdown() { + for { + select { + case b.container.refreshCh <- time.Now(): + case <-b.done: + return + } + } +} + +func (b *Bar) wSyncTable() [][]chan int { + select { + case b.operateState <- func(s *bState) { b.syncTableCh <- s.wSyncTable() }: + return <-b.syncTableCh + case <-b.done: + return b.cacheState.wSyncTable() + } +} + +func (s *bState) draw(stat decor.Statistics) io.Reader { + nlr := strings.NewReader("\n") + tw := stat.AvailableWidth + for _, d := range s.pDecorators { + str := d.Decor(stat) + stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) + s.bufP.WriteString(str) + } + if stat.AvailableWidth < 1 { + trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufP.String()), tw, "…")) + s.bufP.Reset() + return io.MultiReader(trunc, nlr) + } + + if !s.trimSpace && stat.AvailableWidth > 1 { + stat.AvailableWidth -= 2 + s.bufB.WriteByte(' ') + defer s.bufB.WriteByte(' ') + } + + tw = stat.AvailableWidth + for _, d := range s.aDecorators { + str := d.Decor(stat) + stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) + s.bufA.WriteString(str) + } + if stat.AvailableWidth < 1 { + trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufA.String()), tw, "…")) + s.bufA.Reset() + return io.MultiReader(s.bufP, s.bufB, trunc, nlr) + } + + s.filler.Fill(s.bufB, s.reqWidth, stat) + + return io.MultiReader(s.bufP, s.bufB, s.bufA, nlr) +} + +func (s *bState) wSyncTable() [][]chan int { + columns := make([]chan int, 0, len(s.pDecorators)+len(s.aDecorators)) + var pCount int + for _, d := range s.pDecorators { + if ch, ok := d.Sync(); ok { + columns = append(columns, ch) + pCount++ + } + } + var aCount int + for _, d := range s.aDecorators { + if ch, ok := d.Sync(); ok { + columns = append(columns, ch) + aCount++ + } + } + table := make([][]chan int, 2) + table[0] = columns[0:pCount] + table[1] = columns[pCount : pCount+aCount : pCount+aCount] + return table +} + +func newStatistics(tw int, s *bState) decor.Statistics { + return decor.Statistics{ + ID: s.id, + AvailableWidth: tw, + Total: s.total, + Current: s.current, + Refill: s.refill, + Completed: s.completeFlushed, + } +} + +func extractBaseDecorator(d decor.Decorator) decor.Decorator { + if d, ok := d.(decor.Wrapper); ok { + return extractBaseDecorator(d.Base()) + } + return d +} + +func ewmaIterationUpdate(done bool, s *bState, dur time.Duration) { + if !done && !s.iterated { + panic("increment required before ewma iteration update") + } else { + s.iterated = false + } + for _, d := range s.ewmaDecorators { + d.EwmaUpdate(s.lastN, dur) + } +} + +func makePanicExtender(p interface{}) extenderFunc { + pstr := fmt.Sprint(p) + stack := debug.Stack() + stackLines := bytes.Count(stack, []byte("\n")) + return func(_ io.Reader, _ int, st decor.Statistics) (io.Reader, int) { + mr := io.MultiReader( + strings.NewReader(runewidth.Truncate(pstr, st.AvailableWidth, "…")), + strings.NewReader(fmt.Sprintf("\n%#v\n", st)), + bytes.NewReader(stack), + ) + return mr, stackLines + 1 + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/bar_filler.go b/vendor/github.com/vbauerster/mpb/v7/bar_filler.go new file mode 100644 index 000000000..a69087c47 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/bar_filler.go @@ -0,0 +1,39 @@ +package mpb + +import ( + "io" + + "github.com/vbauerster/mpb/v7/decor" +) + +// BarFiller interface. +// Bar (without decorators) renders itself by calling BarFiller's Fill method. +// +// reqWidth is requested width, set by `func WithWidth(int) ContainerOption`. +// If not set, it defaults to terminal width. +// +// Default implementations can be obtained via: +// +// func NewBarFiller(BarStyle()) BarFiller +// func NewBarFiller(SpinnerStyle()) BarFiller +// +type BarFiller interface { + Fill(w io.Writer, reqWidth int, stat decor.Statistics) +} + +// BarFillerBuilder interface. +type BarFillerBuilder interface { + Build() BarFiller +} + +// BarFillerFunc is function type adapter to convert function into BarFiller. +type BarFillerFunc func(w io.Writer, reqWidth int, stat decor.Statistics) + +func (f BarFillerFunc) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { + f(w, reqWidth, stat) +} + +// NewBarFiller constructs a BarFiller from provided BarFillerBuilder. +func NewBarFiller(b BarFillerBuilder) BarFiller { + return b.Build() +} diff --git a/vendor/github.com/vbauerster/mpb/v7/bar_filler_bar.go b/vendor/github.com/vbauerster/mpb/v7/bar_filler_bar.go new file mode 100644 index 000000000..e30d4921c --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/bar_filler_bar.go @@ -0,0 +1,219 @@ +package mpb + +import ( + "io" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" + "github.com/vbauerster/mpb/v7/decor" + "github.com/vbauerster/mpb/v7/internal" +) + +const ( + iLbound = iota + iRbound + iFiller + iRefiller + iPadding + components +) + +// BarStyleComposer interface. +type BarStyleComposer interface { + BarFillerBuilder + Lbound(string) BarStyleComposer + Rbound(string) BarStyleComposer + Filler(string) BarStyleComposer + Refiller(string) BarStyleComposer + Padding(string) BarStyleComposer + Tip(...string) BarStyleComposer + Reverse() BarStyleComposer +} + +type bFiller struct { + components [components]*component + tip struct { + count uint + frames []*component + } + flush func(dst io.Writer, filling, padding [][]byte) +} + +type component struct { + width int + bytes []byte +} + +type barStyle struct { + lbound string + rbound string + filler string + refiller string + padding string + tip []string + rev bool +} + +// BarStyle constructs default bar style which can be altered via +// BarStyleComposer interface. +func BarStyle() BarStyleComposer { + return &barStyle{ + lbound: "[", + rbound: "]", + filler: "=", + refiller: "+", + padding: "-", + tip: []string{">"}, + } +} + +func (s *barStyle) Lbound(bound string) BarStyleComposer { + s.lbound = bound + return s +} + +func (s *barStyle) Rbound(bound string) BarStyleComposer { + s.rbound = bound + return s +} + +func (s *barStyle) Filler(filler string) BarStyleComposer { + s.filler = filler + return s +} + +func (s *barStyle) Refiller(refiller string) BarStyleComposer { + s.refiller = refiller + return s +} + +func (s *barStyle) Padding(padding string) BarStyleComposer { + s.padding = padding + return s +} + +func (s *barStyle) Tip(tip ...string) BarStyleComposer { + if len(tip) != 0 { + s.tip = append(s.tip[:0], tip...) + } + return s +} + +func (s *barStyle) Reverse() BarStyleComposer { + s.rev = true + return s +} + +func (s *barStyle) Build() BarFiller { + bf := new(bFiller) + if s.rev { + bf.flush = func(dst io.Writer, filling, padding [][]byte) { + flush(dst, padding, filling) + } + } else { + bf.flush = flush + } + bf.components[iLbound] = &component{ + width: runewidth.StringWidth(stripansi.Strip(s.lbound)), + bytes: []byte(s.lbound), + } + bf.components[iRbound] = &component{ + width: runewidth.StringWidth(stripansi.Strip(s.rbound)), + bytes: []byte(s.rbound), + } + bf.components[iFiller] = &component{ + width: runewidth.StringWidth(stripansi.Strip(s.filler)), + bytes: []byte(s.filler), + } + bf.components[iRefiller] = &component{ + width: runewidth.StringWidth(stripansi.Strip(s.refiller)), + bytes: []byte(s.refiller), + } + bf.components[iPadding] = &component{ + width: runewidth.StringWidth(stripansi.Strip(s.padding)), + bytes: []byte(s.padding), + } + bf.tip.frames = make([]*component, len(s.tip)) + for i, t := range s.tip { + bf.tip.frames[i] = &component{ + width: runewidth.StringWidth(stripansi.Strip(t)), + bytes: []byte(t), + } + } + return bf +} + +func (s *bFiller) Fill(w io.Writer, width int, stat decor.Statistics) { + width = internal.CheckRequestedWidth(width, stat.AvailableWidth) + brackets := s.components[iLbound].width + s.components[iRbound].width + if width < brackets { + return + } + // don't count brackets as progress + width -= brackets + + w.Write(s.components[iLbound].bytes) + defer w.Write(s.components[iRbound].bytes) + + curWidth := int(internal.PercentageRound(stat.Total, stat.Current, width)) + refWidth, filled := 0, curWidth + filling := make([][]byte, 0, curWidth) + + if curWidth > 0 && curWidth != width { + tipFrame := s.tip.frames[s.tip.count%uint(len(s.tip.frames))] + filling = append(filling, tipFrame.bytes) + curWidth -= tipFrame.width + s.tip.count++ + } + + if stat.Refill > 0 && curWidth > 0 { + refWidth = int(internal.PercentageRound(stat.Total, int64(stat.Refill), width)) + if refWidth > curWidth { + refWidth = curWidth + } + curWidth -= refWidth + } + + for curWidth > 0 && curWidth >= s.components[iFiller].width { + filling = append(filling, s.components[iFiller].bytes) + curWidth -= s.components[iFiller].width + if s.components[iFiller].width == 0 { + break + } + } + + for refWidth > 0 && refWidth >= s.components[iRefiller].width { + filling = append(filling, s.components[iRefiller].bytes) + refWidth -= s.components[iRefiller].width + if s.components[iRefiller].width == 0 { + break + } + } + + filled -= curWidth + refWidth + padWidth := width - filled + padding := make([][]byte, 0, padWidth) + for padWidth > 0 && padWidth >= s.components[iPadding].width { + padding = append(padding, s.components[iPadding].bytes) + padWidth -= s.components[iPadding].width + if s.components[iPadding].width == 0 { + break + } + } + + for padWidth > 0 { + padding = append(padding, []byte("…")) + padWidth-- + } + + s.flush(w, filling, padding) +} + +func flush(dst io.Writer, filling, padding [][]byte) { + for i := len(filling) - 1; i >= 0; i-- { + dst.Write(filling[i]) + } + for i := 0; i < len(padding); i++ { + dst.Write(padding[i]) + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/bar_filler_spinner.go b/vendor/github.com/vbauerster/mpb/v7/bar_filler_spinner.go new file mode 100644 index 000000000..58ae1c532 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/bar_filler_spinner.go @@ -0,0 +1,87 @@ +package mpb + +import ( + "io" + "strings" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" + "github.com/vbauerster/mpb/v7/decor" + "github.com/vbauerster/mpb/v7/internal" +) + +const ( + positionLeft = 1 + iota + positionRight +) + +// SpinnerStyleComposer interface. +type SpinnerStyleComposer interface { + BarFillerBuilder + PositionLeft() SpinnerStyleComposer + PositionRight() SpinnerStyleComposer +} + +type sFiller struct { + count uint + position uint + frames []string +} + +type spinnerStyle struct { + position uint + frames []string +} + +// SpinnerStyle constructs default spinner style which can be altered via +// SpinnerStyleComposer interface. +func SpinnerStyle(frames ...string) SpinnerStyleComposer { + ss := new(spinnerStyle) + if len(frames) != 0 { + ss.frames = append(ss.frames, frames...) + } else { + ss.frames = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + } + return ss +} + +func (s *spinnerStyle) PositionLeft() SpinnerStyleComposer { + s.position = positionLeft + return s +} + +func (s *spinnerStyle) PositionRight() SpinnerStyleComposer { + s.position = positionRight + return s +} + +func (s *spinnerStyle) Build() BarFiller { + sf := &sFiller{ + position: s.position, + frames: s.frames, + } + return sf +} + +func (s *sFiller) Fill(w io.Writer, width int, stat decor.Statistics) { + width = internal.CheckRequestedWidth(width, stat.AvailableWidth) + + frame := s.frames[s.count%uint(len(s.frames))] + frameWidth := runewidth.StringWidth(stripansi.Strip(frame)) + + if width < frameWidth { + return + } + + rest := width - frameWidth + switch s.position { + case positionLeft: + io.WriteString(w, frame+strings.Repeat(" ", rest)) + case positionRight: + io.WriteString(w, strings.Repeat(" ", rest)+frame) + default: + str := strings.Repeat(" ", rest/2) + frame + strings.Repeat(" ", rest/2+rest%2) + io.WriteString(w, str) + } + s.count++ +} diff --git a/vendor/github.com/vbauerster/mpb/v7/bar_option.go b/vendor/github.com/vbauerster/mpb/v7/bar_option.go new file mode 100644 index 000000000..46b7de0bf --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/bar_option.go @@ -0,0 +1,153 @@ +package mpb + +import ( + "bytes" + "io" + + "github.com/vbauerster/mpb/v7/decor" + "github.com/vbauerster/mpb/v7/internal" +) + +// BarOption is a func option to alter default behavior of a bar. +type BarOption func(*bState) + +func (s *bState) addDecorators(dest *[]decor.Decorator, decorators ...decor.Decorator) { + type mergeWrapper interface { + MergeUnwrap() []decor.Decorator + } + for _, decorator := range decorators { + if mw, ok := decorator.(mergeWrapper); ok { + *dest = append(*dest, mw.MergeUnwrap()...) + } + *dest = append(*dest, decorator) + } +} + +// AppendDecorators let you inject decorators to the bar's right side. +func AppendDecorators(decorators ...decor.Decorator) BarOption { + return func(s *bState) { + s.addDecorators(&s.aDecorators, decorators...) + } +} + +// PrependDecorators let you inject decorators to the bar's left side. +func PrependDecorators(decorators ...decor.Decorator) BarOption { + return func(s *bState) { + s.addDecorators(&s.pDecorators, decorators...) + } +} + +// BarID sets bar id. +func BarID(id int) BarOption { + return func(s *bState) { + s.id = id + } +} + +// BarWidth sets bar width independent of the container. +func BarWidth(width int) BarOption { + return func(s *bState) { + s.reqWidth = width + } +} + +// BarQueueAfter queues this (being constructed) bar to relplace +// runningBar after it has been completed. +func BarQueueAfter(runningBar *Bar) BarOption { + if runningBar == nil { + return nil + } + return func(s *bState) { + s.runningBar = runningBar + } +} + +// BarRemoveOnComplete removes both bar's filler and its decorators +// on complete event. +func BarRemoveOnComplete() BarOption { + return func(s *bState) { + s.dropOnComplete = true + } +} + +// BarFillerClearOnComplete clears bar's filler on complete event. +// It's shortcut for BarFillerOnComplete(""). +func BarFillerClearOnComplete() BarOption { + return BarFillerOnComplete("") +} + +// BarFillerOnComplete replaces bar's filler with message, on complete event. +func BarFillerOnComplete(message string) BarOption { + return BarFillerMiddleware(func(base BarFiller) BarFiller { + return BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) { + if st.Completed { + io.WriteString(w, message) + } else { + base.Fill(w, reqWidth, st) + } + }) + }) +} + +// BarFillerMiddleware provides a way to augment the underlying BarFiller. +func BarFillerMiddleware(middle func(BarFiller) BarFiller) BarOption { + return func(s *bState) { + s.middleware = middle + } +} + +// BarPriority sets bar's priority. Zero is highest priority, i.e. bar +// will be on top. If `BarReplaceOnComplete` option is supplied, this +// option is ignored. +func BarPriority(priority int) BarOption { + return func(s *bState) { + s.priority = priority + } +} + +// BarExtender provides a way to extend bar to the next new line. +func BarExtender(filler BarFiller) BarOption { + if filler == nil { + return nil + } + return func(s *bState) { + s.extender = makeExtenderFunc(filler) + } +} + +func makeExtenderFunc(filler BarFiller) extenderFunc { + buf := new(bytes.Buffer) + return func(r io.Reader, reqWidth int, st decor.Statistics) (io.Reader, int) { + filler.Fill(buf, reqWidth, st) + return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), []byte("\n")) + } +} + +// BarFillerTrim removes leading and trailing space around the underlying BarFiller. +func BarFillerTrim() BarOption { + return func(s *bState) { + s.trimSpace = true + } +} + +// BarNoPop disables bar pop out of container. Effective when +// PopCompletedMode of container is enabled. +func BarNoPop() BarOption { + return func(s *bState) { + s.noPop = true + } +} + +// BarOptional will invoke provided option only when pick is true. +func BarOptional(option BarOption, pick bool) BarOption { + return BarOptOn(option, internal.Predicate(pick)) +} + +// BarOptOn will invoke provided option only when higher order predicate +// evaluates to true. +func BarOptOn(option BarOption, predicate func() bool) BarOption { + if predicate() { + return option + } + return nil +} diff --git a/vendor/github.com/vbauerster/mpb/v7/container_option.go b/vendor/github.com/vbauerster/mpb/v7/container_option.go new file mode 100644 index 000000000..e4254f662 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/container_option.go @@ -0,0 +1,112 @@ +package mpb + +import ( + "io" + "io/ioutil" + "sync" + "time" + + "github.com/vbauerster/mpb/v7/internal" +) + +// ContainerOption is a func option to alter default behavior of a bar +// container. Container term refers to a Progress struct which can +// hold one or more Bars. +type ContainerOption func(*pState) + +// WithWaitGroup provides means to have a single joint point. If +// *sync.WaitGroup is provided, you can safely call just p.Wait() +// without calling Wait() on provided *sync.WaitGroup. Makes sense +// when there are more than one bar to render. +func WithWaitGroup(wg *sync.WaitGroup) ContainerOption { + return func(s *pState) { + s.uwg = wg + } +} + +// WithWidth sets container width. If not set it defaults to terminal +// width. A bar added to the container will inherit its width, unless +// overridden by `func BarWidth(int) BarOption`. +func WithWidth(width int) ContainerOption { + return func(s *pState) { + s.reqWidth = width + } +} + +// WithRefreshRate overrides default 120ms refresh rate. +func WithRefreshRate(d time.Duration) ContainerOption { + return func(s *pState) { + s.rr = d + } +} + +// WithManualRefresh disables internal auto refresh time.Ticker. +// Refresh will occur upon receive value from provided ch. +func WithManualRefresh(ch <-chan interface{}) ContainerOption { + return func(s *pState) { + s.externalRefresh = ch + } +} + +// WithRenderDelay delays rendering. By default rendering starts as +// soon as bar is added, with this option it's possible to delay +// rendering process by keeping provided chan unclosed. In other words +// rendering will start as soon as provided chan is closed. +func WithRenderDelay(ch <-chan struct{}) ContainerOption { + return func(s *pState) { + s.renderDelay = ch + } +} + +// WithShutdownNotifier provided chanel will be closed, after all bars +// have been rendered. +func WithShutdownNotifier(ch chan struct{}) ContainerOption { + return func(s *pState) { + s.shutdownNotifier = ch + } +} + +// WithOutput overrides default os.Stdout output. Setting it to nil +// will effectively disable auto refresh rate and discard any output, +// useful if you want to disable progress bars with little overhead. +func WithOutput(w io.Writer) ContainerOption { + return func(s *pState) { + if w == nil { + s.output = ioutil.Discard + s.outputDiscarded = true + return + } + s.output = w + } +} + +// WithDebugOutput sets debug output. +func WithDebugOutput(w io.Writer) ContainerOption { + if w == nil { + return nil + } + return func(s *pState) { + s.debugOut = w + } +} + +// PopCompletedMode will pop and stop rendering completed bars. +func PopCompletedMode() ContainerOption { + return func(s *pState) { + s.popCompleted = true + } +} + +// ContainerOptional will invoke provided option only when pick is true. +func ContainerOptional(option ContainerOption, pick bool) ContainerOption { + return ContainerOptOn(option, internal.Predicate(pick)) +} + +// ContainerOptOn will invoke provided option only when higher order +// predicate evaluates to true. +func ContainerOptOn(option ContainerOption, predicate func() bool) ContainerOption { + if predicate() { + return option + } + return nil +} diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/doc.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/doc.go new file mode 100644 index 000000000..93c8f8268 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/doc.go @@ -0,0 +1,2 @@ +// Package cwriter is a console writer abstraction for the underlying OS. +package cwriter diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/util_bsd.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_bsd.go new file mode 100644 index 000000000..4e3564ece --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_bsd.go @@ -0,0 +1,7 @@ +// +build darwin dragonfly freebsd netbsd openbsd + +package cwriter + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TIOCGETA diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/util_linux.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_linux.go new file mode 100644 index 000000000..253f12dd2 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_linux.go @@ -0,0 +1,7 @@ +// +build aix linux + +package cwriter + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TCGETS diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/util_solaris.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_solaris.go new file mode 100644 index 000000000..4b29ff5c0 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/util_solaris.go @@ -0,0 +1,7 @@ +// +build solaris + +package cwriter + +import "golang.org/x/sys/unix" + +const ioctlReadTermios = unix.TCGETA diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go new file mode 100644 index 000000000..1ade54761 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go @@ -0,0 +1,84 @@ +package cwriter + +import ( + "bytes" + "errors" + "io" + "os" + "strconv" +) + +// ErrNotTTY not a TeleTYpewriter error. +var ErrNotTTY = errors.New("not a terminal") + +// http://ascii-table.com/ansi-escape-sequences.php +const ( + escOpen = "\x1b[" + cuuAndEd = "A\x1b[J" +) + +// Writer is a buffered the writer that updates the terminal. The +// contents of writer will be flushed when Flush is called. +type Writer struct { + out io.Writer + buf bytes.Buffer + lineCount int + fd int + isTerminal bool +} + +// New returns a new Writer with defaults. +func New(out io.Writer) *Writer { + w := &Writer{out: out} + if f, ok := out.(*os.File); ok { + w.fd = int(f.Fd()) + w.isTerminal = IsTerminal(w.fd) + } + return w +} + +// Flush flushes the underlying buffer. +func (w *Writer) Flush(lineCount int) (err error) { + // some terminals interpret 'cursor up 0' as 'cursor up 1' + if w.lineCount > 0 { + err = w.clearLines() + if err != nil { + return + } + } + w.lineCount = lineCount + _, err = w.buf.WriteTo(w.out) + return +} + +// Write appends the contents of p to the underlying buffer. +func (w *Writer) Write(p []byte) (n int, err error) { + return w.buf.Write(p) +} + +// WriteString writes string to the underlying buffer. +func (w *Writer) WriteString(s string) (n int, err error) { + return w.buf.WriteString(s) +} + +// ReadFrom reads from the provided io.Reader and writes to the +// underlying buffer. +func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { + return w.buf.ReadFrom(r) +} + +// GetWidth returns width of underlying terminal. +func (w *Writer) GetWidth() (int, error) { + if !w.isTerminal { + return -1, ErrNotTTY + } + tw, _, err := GetSize(w.fd) + return tw, err +} + +func (w *Writer) ansiCuuAndEd() (err error) { + buf := make([]byte, 8) + buf = strconv.AppendInt(buf[:copy(buf, escOpen)], int64(w.lineCount), 10) + _, err = w.out.Write(append(buf, cuuAndEd...)) + return +} diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_posix.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_posix.go new file mode 100644 index 000000000..f54a5d06b --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_posix.go @@ -0,0 +1,26 @@ +// +build !windows + +package cwriter + +import ( + "golang.org/x/sys/unix" +) + +func (w *Writer) clearLines() error { + return w.ansiCuuAndEd() +} + +// GetSize returns the dimensions of the given terminal. +func GetSize(fd int) (width, height int, err error) { + ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ) + if err != nil { + return -1, -1, err + } + return int(ws.Col), int(ws.Row), nil +} + +// IsTerminal returns whether the given file descriptor is a terminal. +func IsTerminal(fd int) bool { + _, err := unix.IoctlGetTermios(fd, ioctlReadTermios) + return err == nil +} diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go new file mode 100644 index 000000000..1a69c81ac --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go @@ -0,0 +1,73 @@ +// +build windows + +package cwriter + +import ( + "unsafe" + + "golang.org/x/sys/windows" +) + +var kernel32 = windows.NewLazySystemDLL("kernel32.dll") + +var ( + procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") + procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW") +) + +func (w *Writer) clearLines() error { + if !w.isTerminal { + // hope it's cygwin or similar + return w.ansiCuuAndEd() + } + + var info windows.ConsoleScreenBufferInfo + if err := windows.GetConsoleScreenBufferInfo(windows.Handle(w.fd), &info); err != nil { + return err + } + + info.CursorPosition.Y -= int16(w.lineCount) + if info.CursorPosition.Y < 0 { + info.CursorPosition.Y = 0 + } + _, _, _ = procSetConsoleCursorPosition.Call( + uintptr(w.fd), + uintptr(uint32(uint16(info.CursorPosition.Y))<<16|uint32(uint16(info.CursorPosition.X))), + ) + + // clear the lines + cursor := &windows.Coord{ + X: info.Window.Left, + Y: info.CursorPosition.Y, + } + count := uint32(info.Size.X) * uint32(w.lineCount) + _, _, _ = procFillConsoleOutputCharacter.Call( + uintptr(w.fd), + uintptr(' '), + uintptr(count), + *(*uintptr)(unsafe.Pointer(cursor)), + uintptr(unsafe.Pointer(new(uint32))), + ) + return nil +} + +// GetSize returns the visible dimensions of the given terminal. +// +// These dimensions don't include any scrollback buffer height. +func GetSize(fd int) (width, height int, err error) { + var info windows.ConsoleScreenBufferInfo + if err := windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info); err != nil { + return 0, 0, err + } + // terminal.GetSize from crypto/ssh adds "+ 1" to both width and height: + // https://go.googlesource.com/crypto/+/refs/heads/release-branch.go1.14/ssh/terminal/util_windows.go#75 + // but looks like this is a root cause of issue #66, so removing both "+ 1" have fixed it. + return int(info.Window.Right - info.Window.Left), int(info.Window.Bottom - info.Window.Top), nil +} + +// IsTerminal returns whether the given file descriptor is a terminal. +func IsTerminal(fd int) bool { + var st uint32 + err := windows.GetConsoleMode(windows.Handle(fd), &st) + return err == nil +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/any.go b/vendor/github.com/vbauerster/mpb/v7/decor/any.go new file mode 100644 index 000000000..39518f594 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/any.go @@ -0,0 +1,21 @@ +package decor + +// Any decorator displays text, that can be changed during decorator's +// lifetime via provided DecorFunc. +// +// `fn` DecorFunc callback +// +// `wcc` optional WC config +// +func Any(fn DecorFunc, wcc ...WC) Decorator { + return &any{initWC(wcc...), fn} +} + +type any struct { + WC + fn DecorFunc +} + +func (d *any) Decor(s Statistics) string { + return d.FormatMsg(d.fn(s)) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/counters.go b/vendor/github.com/vbauerster/mpb/v7/decor/counters.go new file mode 100644 index 000000000..4a5343d41 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/counters.go @@ -0,0 +1,243 @@ +package decor + +import ( + "fmt" + "strings" +) + +const ( + _ = iota + UnitKiB + UnitKB +) + +// CountersNoUnit is a wrapper around Counters with no unit param. +func CountersNoUnit(pairFmt string, wcc ...WC) Decorator { + return Counters(0, pairFmt, wcc...) +} + +// CountersKibiByte is a wrapper around Counters with predefined unit +// UnitKiB (bytes/1024). +func CountersKibiByte(pairFmt string, wcc ...WC) Decorator { + return Counters(UnitKiB, pairFmt, wcc...) +} + +// CountersKiloByte is a wrapper around Counters with predefined unit +// UnitKB (bytes/1000). +func CountersKiloByte(pairFmt string, wcc ...WC) Decorator { + return Counters(UnitKB, pairFmt, wcc...) +} + +// Counters decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `pairFmt` printf compatible verbs for current and total pair +// +// `wcc` optional WC config +// +// pairFmt example if unit=UnitKB: +// +// pairFmt="%.1f / %.1f" output: "1.0MB / 12.0MB" +// pairFmt="% .1f / % .1f" output: "1.0 MB / 12.0 MB" +// pairFmt="%d / %d" output: "1MB / 12MB" +// pairFmt="% d / % d" output: "1 MB / 12 MB" +// +func Counters(unit int, pairFmt string, wcc ...WC) Decorator { + producer := func(unit int, pairFmt string) DecorFunc { + if pairFmt == "" { + pairFmt = "%d / %d" + } else if strings.Count(pairFmt, "%") != 2 { + panic("expected pairFmt with exactly 2 verbs") + } + switch unit { + case UnitKiB: + return func(s Statistics) string { + return fmt.Sprintf(pairFmt, SizeB1024(s.Current), SizeB1024(s.Total)) + } + case UnitKB: + return func(s Statistics) string { + return fmt.Sprintf(pairFmt, SizeB1000(s.Current), SizeB1000(s.Total)) + } + default: + return func(s Statistics) string { + return fmt.Sprintf(pairFmt, s.Current, s.Total) + } + } + } + return Any(producer(unit, pairFmt), wcc...) +} + +// TotalNoUnit is a wrapper around Total with no unit param. +func TotalNoUnit(format string, wcc ...WC) Decorator { + return Total(0, format, wcc...) +} + +// TotalKibiByte is a wrapper around Total with predefined unit +// UnitKiB (bytes/1024). +func TotalKibiByte(format string, wcc ...WC) Decorator { + return Total(UnitKiB, format, wcc...) +} + +// TotalKiloByte is a wrapper around Total with predefined unit +// UnitKB (bytes/1000). +func TotalKiloByte(format string, wcc ...WC) Decorator { + return Total(UnitKB, format, wcc...) +} + +// Total decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `format` printf compatible verb for Total +// +// `wcc` optional WC config +// +// format example if unit=UnitKiB: +// +// format="%.1f" output: "12.0MiB" +// format="% .1f" output: "12.0 MiB" +// format="%d" output: "12MiB" +// format="% d" output: "12 MiB" +// +func Total(unit int, format string, wcc ...WC) Decorator { + producer := func(unit int, format string) DecorFunc { + if format == "" { + format = "%d" + } else if strings.Count(format, "%") != 1 { + panic("expected format with exactly 1 verb") + } + + switch unit { + case UnitKiB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1024(s.Total)) + } + case UnitKB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1000(s.Total)) + } + default: + return func(s Statistics) string { + return fmt.Sprintf(format, s.Total) + } + } + } + return Any(producer(unit, format), wcc...) +} + +// CurrentNoUnit is a wrapper around Current with no unit param. +func CurrentNoUnit(format string, wcc ...WC) Decorator { + return Current(0, format, wcc...) +} + +// CurrentKibiByte is a wrapper around Current with predefined unit +// UnitKiB (bytes/1024). +func CurrentKibiByte(format string, wcc ...WC) Decorator { + return Current(UnitKiB, format, wcc...) +} + +// CurrentKiloByte is a wrapper around Current with predefined unit +// UnitKB (bytes/1000). +func CurrentKiloByte(format string, wcc ...WC) Decorator { + return Current(UnitKB, format, wcc...) +} + +// Current decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `format` printf compatible verb for Current +// +// `wcc` optional WC config +// +// format example if unit=UnitKiB: +// +// format="%.1f" output: "12.0MiB" +// format="% .1f" output: "12.0 MiB" +// format="%d" output: "12MiB" +// format="% d" output: "12 MiB" +// +func Current(unit int, format string, wcc ...WC) Decorator { + producer := func(unit int, format string) DecorFunc { + if format == "" { + format = "%d" + } else if strings.Count(format, "%") != 1 { + panic("expected format with exactly 1 verb") + } + + switch unit { + case UnitKiB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1024(s.Current)) + } + case UnitKB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1000(s.Current)) + } + default: + return func(s Statistics) string { + return fmt.Sprintf(format, s.Current) + } + } + } + return Any(producer(unit, format), wcc...) +} + +// InvertedCurrentNoUnit is a wrapper around InvertedCurrent with no unit param. +func InvertedCurrentNoUnit(format string, wcc ...WC) Decorator { + return InvertedCurrent(0, format, wcc...) +} + +// InvertedCurrentKibiByte is a wrapper around InvertedCurrent with predefined unit +// UnitKiB (bytes/1024). +func InvertedCurrentKibiByte(format string, wcc ...WC) Decorator { + return InvertedCurrent(UnitKiB, format, wcc...) +} + +// InvertedCurrentKiloByte is a wrapper around InvertedCurrent with predefined unit +// UnitKB (bytes/1000). +func InvertedCurrentKiloByte(format string, wcc ...WC) Decorator { + return InvertedCurrent(UnitKB, format, wcc...) +} + +// InvertedCurrent decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `format` printf compatible verb for InvertedCurrent +// +// `wcc` optional WC config +// +// format example if unit=UnitKiB: +// +// format="%.1f" output: "12.0MiB" +// format="% .1f" output: "12.0 MiB" +// format="%d" output: "12MiB" +// format="% d" output: "12 MiB" +// +func InvertedCurrent(unit int, format string, wcc ...WC) Decorator { + producer := func(unit int, format string) DecorFunc { + if format == "" { + format = "%d" + } else if strings.Count(format, "%") != 1 { + panic("expected format with exactly 1 verb") + } + + switch unit { + case UnitKiB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1024(s.Total-s.Current)) + } + case UnitKB: + return func(s Statistics) string { + return fmt.Sprintf(format, SizeB1000(s.Total-s.Current)) + } + default: + return func(s Statistics) string { + return fmt.Sprintf(format, s.Total-s.Current) + } + } + } + return Any(producer(unit, format), wcc...) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/decorator.go b/vendor/github.com/vbauerster/mpb/v7/decor/decorator.go new file mode 100644 index 000000000..e81fae367 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/decorator.go @@ -0,0 +1,191 @@ +package decor + +import ( + "fmt" + "time" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" +) + +const ( + // DidentRight bit specifies identation direction. + // |foo |b | With DidentRight + // | foo| b| Without DidentRight + DidentRight = 1 << iota + + // DextraSpace bit adds extra space, makes sense with DSyncWidth only. + // When DidentRight bit set, the space will be added to the right, + // otherwise to the left. + DextraSpace + + // DSyncWidth bit enables same column width synchronization. + // Effective with multiple bars only. + DSyncWidth + + // DSyncWidthR is shortcut for DSyncWidth|DidentRight + DSyncWidthR = DSyncWidth | DidentRight + + // DSyncSpace is shortcut for DSyncWidth|DextraSpace + DSyncSpace = DSyncWidth | DextraSpace + + // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight + DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight +) + +// TimeStyle enum. +type TimeStyle int + +// TimeStyle kinds. +const ( + ET_STYLE_GO TimeStyle = iota + ET_STYLE_HHMMSS + ET_STYLE_HHMM + ET_STYLE_MMSS +) + +// Statistics consists of progress related statistics, that Decorator +// may need. +type Statistics struct { + ID int + AvailableWidth int + Total int64 + Current int64 + Refill int64 + Completed bool +} + +// Decorator interface. +// Most of the time there is no need to implement this interface +// manually, as decor package already provides a wide range of decorators +// which implement this interface. If however built-in decorators don't +// meet your needs, you're free to implement your own one by implementing +// this particular interface. The easy way to go is to convert a +// `DecorFunc` into a `Decorator` interface by using provided +// `func Any(DecorFunc, ...WC) Decorator`. +type Decorator interface { + Configurator + Synchronizer + Decor(Statistics) string +} + +// DecorFunc func type. +// To be used with `func Any`(DecorFunc, ...WC) Decorator`. +type DecorFunc func(Statistics) string + +// Synchronizer interface. +// All decorators implement this interface implicitly. Its Sync +// method exposes width sync channel, if DSyncWidth bit is set. +type Synchronizer interface { + Sync() (chan int, bool) +} + +// Configurator interface. +type Configurator interface { + GetConf() WC + SetConf(WC) +} + +// Wrapper interface. +// If you're implementing custom Decorator by wrapping a built-in one, +// it is necessary to implement this interface to retain functionality +// of built-in Decorator. +type Wrapper interface { + Base() Decorator +} + +// EwmaDecorator interface. +// EWMA based decorators should implement this one. +type EwmaDecorator interface { + EwmaUpdate(int64, time.Duration) +} + +// AverageDecorator interface. +// Average decorators should implement this interface to provide start +// time adjustment facility, for resume-able tasks. +type AverageDecorator interface { + AverageAdjust(time.Time) +} + +// ShutdownListener interface. +// If decorator needs to be notified once upon bar shutdown event, so +// this is the right interface to implement. +type ShutdownListener interface { + Shutdown() +} + +// Global convenience instances of WC with sync width bit set. +// To be used with multiple bars only, i.e. not effective for single bar usage. +var ( + WCSyncWidth = WC{C: DSyncWidth} + WCSyncWidthR = WC{C: DSyncWidthR} + WCSyncSpace = WC{C: DSyncSpace} + WCSyncSpaceR = WC{C: DSyncSpaceR} +) + +// WC is a struct with two public fields W and C, both of int type. +// W represents width and C represents bit set of width related config. +// A decorator should embed WC, to enable width synchronization. +type WC struct { + W int + C int + fill func(s string, w int) string + wsync chan int +} + +// FormatMsg formats final message according to WC.W and WC.C. +// Should be called by any Decorator implementation. +func (wc *WC) FormatMsg(msg string) string { + pureWidth := runewidth.StringWidth(msg) + stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) + maxCell := wc.W + if (wc.C & DSyncWidth) != 0 { + cellCount := stripWidth + if (wc.C & DextraSpace) != 0 { + cellCount++ + } + wc.wsync <- cellCount + maxCell = <-wc.wsync + } + return wc.fill(msg, maxCell+(pureWidth-stripWidth)) +} + +// Init initializes width related config. +func (wc *WC) Init() WC { + wc.fill = runewidth.FillLeft + if (wc.C & DidentRight) != 0 { + wc.fill = runewidth.FillRight + } + if (wc.C & DSyncWidth) != 0 { + // it's deliberate choice to override wsync on each Init() call, + // this way globals like WCSyncSpace can be reused + wc.wsync = make(chan int) + } + return *wc +} + +// Sync is implementation of Synchronizer interface. +func (wc *WC) Sync() (chan int, bool) { + if (wc.C&DSyncWidth) != 0 && wc.wsync == nil { + panic(fmt.Sprintf("%T is not initialized", wc)) + } + return wc.wsync, (wc.C & DSyncWidth) != 0 +} + +// GetConf is implementation of Configurator interface. +func (wc *WC) GetConf() WC { + return *wc +} + +// SetConf is implementation of Configurator interface. +func (wc *WC) SetConf(conf WC) { + *wc = conf.Init() +} + +func initWC(wcc ...WC) WC { + var wc WC + for _, nwc := range wcc { + wc = nwc + } + return wc.Init() +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/doc.go b/vendor/github.com/vbauerster/mpb/v7/decor/doc.go new file mode 100644 index 000000000..4e4299307 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/doc.go @@ -0,0 +1,19 @@ +// Package decor provides common decorators for "github.com/vbauerster/mpb/v7" module. +// +// Some decorators returned by this package might have a closure state. It is ok to use +// decorators concurrently, unless you share the same decorator among multiple +// *mpb.Bar instances. To avoid data races, create new decorator per *mpb.Bar instance. +// +// Don't: +// +// p := mpb.New() +// name := decor.Name("bar") +// p.AddBar(100, mpb.AppendDecorators(name)) +// p.AddBar(100, mpb.AppendDecorators(name)) +// +// Do: +// +// p := mpb.New() +// p.AddBar(100, mpb.AppendDecorators(decor.Name("bar1"))) +// p.AddBar(100, mpb.AppendDecorators(decor.Name("bar2"))) +package decor diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/elapsed.go b/vendor/github.com/vbauerster/mpb/v7/decor/elapsed.go new file mode 100644 index 000000000..e389f1581 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/elapsed.go @@ -0,0 +1,35 @@ +package decor + +import ( + "time" +) + +// Elapsed decorator. It's wrapper of NewElapsed. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `wcc` optional WC config +// +func Elapsed(style TimeStyle, wcc ...WC) Decorator { + return NewElapsed(style, time.Now(), wcc...) +} + +// NewElapsed returns elapsed time decorator. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `startTime` start time +// +// `wcc` optional WC config +// +func NewElapsed(style TimeStyle, startTime time.Time, wcc ...WC) Decorator { + var msg string + producer := chooseTimeProducer(style) + fn := func(s Statistics) string { + if !s.Completed { + msg = producer(time.Since(startTime)) + } + return msg + } + return Any(fn, wcc...) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/eta.go b/vendor/github.com/vbauerster/mpb/v7/decor/eta.go new file mode 100644 index 000000000..d03caa735 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/eta.go @@ -0,0 +1,203 @@ +package decor + +import ( + "fmt" + "math" + "time" + + "github.com/VividCortex/ewma" +) + +// TimeNormalizer interface. Implementors could be passed into +// MovingAverageETA, in order to affect i.e. normalize its output. +type TimeNormalizer interface { + Normalize(time.Duration) time.Duration +} + +// TimeNormalizerFunc is function type adapter to convert function +// into TimeNormalizer. +type TimeNormalizerFunc func(time.Duration) time.Duration + +func (f TimeNormalizerFunc) Normalize(src time.Duration) time.Duration { + return f(src) +} + +// EwmaETA exponential-weighted-moving-average based ETA decorator. +// For this decorator to work correctly you have to measure each +// iteration's duration and pass it to the +// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment. +func EwmaETA(style TimeStyle, age float64, wcc ...WC) Decorator { + var average ewma.MovingAverage + if age == 0 { + average = ewma.NewMovingAverage() + } else { + average = ewma.NewMovingAverage(age) + } + return MovingAverageETA(style, NewThreadSafeMovingAverage(average), nil, wcc...) +} + +// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `average` implementation of MovingAverage interface +// +// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer] +// +// `wcc` optional WC config +// +func MovingAverageETA(style TimeStyle, average ewma.MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator { + d := &movingAverageETA{ + WC: initWC(wcc...), + average: average, + normalizer: normalizer, + producer: chooseTimeProducer(style), + } + return d +} + +type movingAverageETA struct { + WC + average ewma.MovingAverage + normalizer TimeNormalizer + producer func(time.Duration) string +} + +func (d *movingAverageETA) Decor(s Statistics) string { + v := math.Round(d.average.Value()) + remaining := time.Duration((s.Total - s.Current) * int64(v)) + if d.normalizer != nil { + remaining = d.normalizer.Normalize(remaining) + } + return d.FormatMsg(d.producer(remaining)) +} + +func (d *movingAverageETA) EwmaUpdate(n int64, dur time.Duration) { + durPerItem := float64(dur) / float64(n) + if math.IsInf(durPerItem, 0) || math.IsNaN(durPerItem) { + return + } + d.average.Add(durPerItem) +} + +// AverageETA decorator. It's wrapper of NewAverageETA. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `wcc` optional WC config +// +func AverageETA(style TimeStyle, wcc ...WC) Decorator { + return NewAverageETA(style, time.Now(), nil, wcc...) +} + +// NewAverageETA decorator with user provided start time. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `startTime` start time +// +// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer] +// +// `wcc` optional WC config +// +func NewAverageETA(style TimeStyle, startTime time.Time, normalizer TimeNormalizer, wcc ...WC) Decorator { + d := &averageETA{ + WC: initWC(wcc...), + startTime: startTime, + normalizer: normalizer, + producer: chooseTimeProducer(style), + } + return d +} + +type averageETA struct { + WC + startTime time.Time + normalizer TimeNormalizer + producer func(time.Duration) string +} + +func (d *averageETA) Decor(s Statistics) string { + var remaining time.Duration + if s.Current != 0 { + durPerItem := float64(time.Since(d.startTime)) / float64(s.Current) + durPerItem = math.Round(durPerItem) + remaining = time.Duration((s.Total - s.Current) * int64(durPerItem)) + if d.normalizer != nil { + remaining = d.normalizer.Normalize(remaining) + } + } + return d.FormatMsg(d.producer(remaining)) +} + +func (d *averageETA) AverageAdjust(startTime time.Time) { + d.startTime = startTime +} + +// MaxTolerateTimeNormalizer returns implementation of TimeNormalizer. +func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer { + var normalized time.Duration + var lastCall time.Time + return TimeNormalizerFunc(func(remaining time.Duration) time.Duration { + if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < time.Minute { + normalized = remaining + lastCall = time.Now() + return remaining + } + normalized -= time.Since(lastCall) + lastCall = time.Now() + return normalized + }) +} + +// FixedIntervalTimeNormalizer returns implementation of TimeNormalizer. +func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer { + var normalized time.Duration + var lastCall time.Time + var count int + return TimeNormalizerFunc(func(remaining time.Duration) time.Duration { + if count == 0 || remaining < time.Minute { + count = updInterval + normalized = remaining + lastCall = time.Now() + return remaining + } + count-- + normalized -= time.Since(lastCall) + lastCall = time.Now() + return normalized + }) +} + +func chooseTimeProducer(style TimeStyle) func(time.Duration) string { + switch style { + case ET_STYLE_HHMMSS: + return func(remaining time.Duration) string { + hours := int64(remaining/time.Hour) % 60 + minutes := int64(remaining/time.Minute) % 60 + seconds := int64(remaining/time.Second) % 60 + return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + } + case ET_STYLE_HHMM: + return func(remaining time.Duration) string { + hours := int64(remaining/time.Hour) % 60 + minutes := int64(remaining/time.Minute) % 60 + return fmt.Sprintf("%02d:%02d", hours, minutes) + } + case ET_STYLE_MMSS: + return func(remaining time.Duration) string { + hours := int64(remaining/time.Hour) % 60 + minutes := int64(remaining/time.Minute) % 60 + seconds := int64(remaining/time.Second) % 60 + if hours > 0 { + return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + } + return fmt.Sprintf("%02d:%02d", minutes, seconds) + } + default: + return func(remaining time.Duration) string { + // strip off nanoseconds + return ((remaining / time.Second) * time.Second).String() + } + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/merge.go b/vendor/github.com/vbauerster/mpb/v7/decor/merge.go new file mode 100644 index 000000000..e41406a64 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/merge.go @@ -0,0 +1,107 @@ +package decor + +import ( + "strings" + + "github.com/acarl005/stripansi" + "github.com/mattn/go-runewidth" +) + +// Merge wraps its decorator argument with intention to sync width +// with several decorators of another bar. Visual example: +// +// +----+--------+---------+--------+ +// | B1 | MERGE(D, P1, Pn) | +// +----+--------+---------+--------+ +// | B2 | D0 | D1 | Dn | +// +----+--------+---------+--------+ +// +func Merge(decorator Decorator, placeholders ...WC) Decorator { + if _, ok := decorator.Sync(); !ok || len(placeholders) == 0 { + return decorator + } + md := &mergeDecorator{ + Decorator: decorator, + wc: decorator.GetConf(), + placeHolders: make([]*placeHolderDecorator, len(placeholders)), + } + decorator.SetConf(WC{}) + for i, wc := range placeholders { + if (wc.C & DSyncWidth) == 0 { + return decorator + } + md.placeHolders[i] = &placeHolderDecorator{wc.Init()} + } + return md +} + +type mergeDecorator struct { + Decorator + wc WC + placeHolders []*placeHolderDecorator +} + +func (d *mergeDecorator) GetConf() WC { + return d.wc +} + +func (d *mergeDecorator) SetConf(conf WC) { + d.wc = conf.Init() +} + +func (d *mergeDecorator) MergeUnwrap() []Decorator { + decorators := make([]Decorator, len(d.placeHolders)) + for i, ph := range d.placeHolders { + decorators[i] = ph + } + return decorators +} + +func (d *mergeDecorator) Sync() (chan int, bool) { + return d.wc.Sync() +} + +func (d *mergeDecorator) Base() Decorator { + return d.Decorator +} + +func (d *mergeDecorator) Decor(s Statistics) string { + msg := d.Decorator.Decor(s) + pureWidth := runewidth.StringWidth(msg) + stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) + cellCount := stripWidth + if (d.wc.C & DextraSpace) != 0 { + cellCount++ + } + + total := runewidth.StringWidth(d.placeHolders[0].FormatMsg("")) + pw := (cellCount - total) / len(d.placeHolders) + rem := (cellCount - total) % len(d.placeHolders) + + var diff int + for i := 1; i < len(d.placeHolders); i++ { + ph := d.placeHolders[i] + width := pw - diff + if (ph.WC.C & DextraSpace) != 0 { + width-- + if width < 0 { + width = 0 + } + } + max := runewidth.StringWidth(ph.FormatMsg(strings.Repeat(" ", width))) + total += max + diff = max - pw + } + + d.wc.wsync <- pw + rem + max := <-d.wc.wsync + return d.wc.fill(msg, max+total+(pureWidth-stripWidth)) +} + +type placeHolderDecorator struct { + WC +} + +func (d *placeHolderDecorator) Decor(Statistics) string { + return "" +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/moving_average.go b/vendor/github.com/vbauerster/mpb/v7/decor/moving_average.go new file mode 100644 index 000000000..50ac9c393 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/moving_average.go @@ -0,0 +1,68 @@ +package decor + +import ( + "sort" + "sync" + + "github.com/VividCortex/ewma" +) + +type threadSafeMovingAverage struct { + ewma.MovingAverage + mu sync.Mutex +} + +func (s *threadSafeMovingAverage) Add(value float64) { + s.mu.Lock() + s.MovingAverage.Add(value) + s.mu.Unlock() +} + +func (s *threadSafeMovingAverage) Value() float64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.MovingAverage.Value() +} + +func (s *threadSafeMovingAverage) Set(value float64) { + s.mu.Lock() + s.MovingAverage.Set(value) + s.mu.Unlock() +} + +// NewThreadSafeMovingAverage converts provided ewma.MovingAverage +// into thread safe ewma.MovingAverage. +func NewThreadSafeMovingAverage(average ewma.MovingAverage) ewma.MovingAverage { + if tsma, ok := average.(*threadSafeMovingAverage); ok { + return tsma + } + return &threadSafeMovingAverage{MovingAverage: average} +} + +type medianWindow [3]float64 + +func (s *medianWindow) Len() int { return len(s) } +func (s *medianWindow) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s *medianWindow) Less(i, j int) bool { return s[i] < s[j] } + +func (s *medianWindow) Add(value float64) { + s[0], s[1] = s[1], s[2] + s[2] = value +} + +func (s *medianWindow) Value() float64 { + tmp := *s + sort.Sort(&tmp) + return tmp[1] +} + +func (s *medianWindow) Set(value float64) { + for i := 0; i < len(s); i++ { + s[i] = value + } +} + +// NewMedian is fixed last 3 samples median MovingAverage. +func NewMedian() ewma.MovingAverage { + return NewThreadSafeMovingAverage(new(medianWindow)) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/name.go b/vendor/github.com/vbauerster/mpb/v7/decor/name.go new file mode 100644 index 000000000..3af311254 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/name.go @@ -0,0 +1,12 @@ +package decor + +// Name decorator displays text that is set once and can't be changed +// during decorator's lifetime. +// +// `str` string to display +// +// `wcc` optional WC config +// +func Name(str string, wcc ...WC) Decorator { + return Any(func(Statistics) string { return str }, wcc...) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/on_complete.go b/vendor/github.com/vbauerster/mpb/v7/decor/on_complete.go new file mode 100644 index 000000000..f46b19aba --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/on_complete.go @@ -0,0 +1,37 @@ +package decor + +// OnComplete returns decorator, which wraps provided decorator, with +// sole purpose to display provided message on complete event. +// +// `decorator` Decorator to wrap +// +// `message` message to display on complete event +// +func OnComplete(decorator Decorator, message string) Decorator { + d := &onCompleteWrapper{ + Decorator: decorator, + msg: message, + } + if md, ok := decorator.(*mergeDecorator); ok { + d.Decorator, md.Decorator = md.Decorator, d + return md + } + return d +} + +type onCompleteWrapper struct { + Decorator + msg string +} + +func (d *onCompleteWrapper) Decor(s Statistics) string { + if s.Completed { + wc := d.GetConf() + return wc.FormatMsg(d.msg) + } + return d.Decorator.Decor(s) +} + +func (d *onCompleteWrapper) Base() Decorator { + return d.Decorator +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/percentage.go b/vendor/github.com/vbauerster/mpb/v7/decor/percentage.go new file mode 100644 index 000000000..2b0a7a956 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/percentage.go @@ -0,0 +1,58 @@ +package decor + +import ( + "fmt" + "io" + "strconv" + + "github.com/vbauerster/mpb/v7/internal" +) + +type percentageType float64 + +func (s percentageType) Format(st fmt.State, verb rune) { + var prec int + switch verb { + case 'd': + case 's': + prec = -1 + default: + if p, ok := st.Precision(); ok { + prec = p + } else { + prec = 6 + } + } + + io.WriteString(st, strconv.FormatFloat(float64(s), 'f', prec, 64)) + + if st.Flag(' ') { + io.WriteString(st, " ") + } + io.WriteString(st, "%") +} + +// Percentage returns percentage decorator. It's a wrapper of NewPercentage. +func Percentage(wcc ...WC) Decorator { + return NewPercentage("% d", wcc...) +} + +// NewPercentage percentage decorator with custom format string. +// +// format examples: +// +// format="%.1f" output: "1.0%" +// format="% .1f" output: "1.0 %" +// format="%d" output: "1%" +// format="% d" output: "1 %" +// +func NewPercentage(format string, wcc ...WC) Decorator { + if format == "" { + format = "% d" + } + f := func(s Statistics) string { + p := internal.Percentage(s.Total, s.Current, 100) + return fmt.Sprintf(format, percentageType(p)) + } + return Any(f, wcc...) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/size_type.go b/vendor/github.com/vbauerster/mpb/v7/decor/size_type.go new file mode 100644 index 000000000..e4b974058 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/size_type.go @@ -0,0 +1,109 @@ +package decor + +import ( + "fmt" + "io" + "math" + "strconv" +) + +//go:generate stringer -type=SizeB1024 -trimprefix=_i +//go:generate stringer -type=SizeB1000 -trimprefix=_ + +const ( + _ib SizeB1024 = iota + 1 + _iKiB SizeB1024 = 1 << (iota * 10) + _iMiB + _iGiB + _iTiB +) + +// SizeB1024 named type, which implements fmt.Formatter interface. It +// adjusts its value according to byte size multiple by 1024 and appends +// appropriate size marker (KiB, MiB, GiB, TiB). +type SizeB1024 int64 + +func (self SizeB1024) Format(st fmt.State, verb rune) { + var prec int + switch verb { + case 'd': + case 's': + prec = -1 + default: + if p, ok := st.Precision(); ok { + prec = p + } else { + prec = 6 + } + } + + var unit SizeB1024 + switch { + case self < _iKiB: + unit = _ib + case self < _iMiB: + unit = _iKiB + case self < _iGiB: + unit = _iMiB + case self < _iTiB: + unit = _iGiB + case self <= math.MaxInt64: + unit = _iTiB + } + + io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64)) + + if st.Flag(' ') { + io.WriteString(st, " ") + } + io.WriteString(st, unit.String()) +} + +const ( + _b SizeB1000 = 1 + _KB SizeB1000 = _b * 1000 + _MB SizeB1000 = _KB * 1000 + _GB SizeB1000 = _MB * 1000 + _TB SizeB1000 = _GB * 1000 +) + +// SizeB1000 named type, which implements fmt.Formatter interface. It +// adjusts its value according to byte size multiple by 1000 and appends +// appropriate size marker (KB, MB, GB, TB). +type SizeB1000 int64 + +func (self SizeB1000) Format(st fmt.State, verb rune) { + var prec int + switch verb { + case 'd': + case 's': + prec = -1 + default: + if p, ok := st.Precision(); ok { + prec = p + } else { + prec = 6 + } + } + + var unit SizeB1000 + switch { + case self < _KB: + unit = _b + case self < _MB: + unit = _KB + case self < _GB: + unit = _MB + case self < _TB: + unit = _GB + case self <= math.MaxInt64: + unit = _TB + } + + io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64)) + + if st.Flag(' ') { + io.WriteString(st, " ") + } + io.WriteString(st, unit.String()) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1000_string.go b/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1000_string.go new file mode 100644 index 000000000..3f32ef715 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1000_string.go @@ -0,0 +1,41 @@ +// Code generated by "stringer -type=SizeB1000 -trimprefix=_"; DO NOT EDIT. + +package decor + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[_b-1] + _ = x[_KB-1000] + _ = x[_MB-1000000] + _ = x[_GB-1000000000] + _ = x[_TB-1000000000000] +} + +const ( + _SizeB1000_name_0 = "b" + _SizeB1000_name_1 = "KB" + _SizeB1000_name_2 = "MB" + _SizeB1000_name_3 = "GB" + _SizeB1000_name_4 = "TB" +) + +func (i SizeB1000) String() string { + switch { + case i == 1: + return _SizeB1000_name_0 + case i == 1000: + return _SizeB1000_name_1 + case i == 1000000: + return _SizeB1000_name_2 + case i == 1000000000: + return _SizeB1000_name_3 + case i == 1000000000000: + return _SizeB1000_name_4 + default: + return "SizeB1000(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1024_string.go b/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1024_string.go new file mode 100644 index 000000000..9fca66cc7 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/sizeb1024_string.go @@ -0,0 +1,41 @@ +// Code generated by "stringer -type=SizeB1024 -trimprefix=_i"; DO NOT EDIT. + +package decor + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[_ib-1] + _ = x[_iKiB-1024] + _ = x[_iMiB-1048576] + _ = x[_iGiB-1073741824] + _ = x[_iTiB-1099511627776] +} + +const ( + _SizeB1024_name_0 = "b" + _SizeB1024_name_1 = "KiB" + _SizeB1024_name_2 = "MiB" + _SizeB1024_name_3 = "GiB" + _SizeB1024_name_4 = "TiB" +) + +func (i SizeB1024) String() string { + switch { + case i == 1: + return _SizeB1024_name_0 + case i == 1024: + return _SizeB1024_name_1 + case i == 1048576: + return _SizeB1024_name_2 + case i == 1073741824: + return _SizeB1024_name_3 + case i == 1099511627776: + return _SizeB1024_name_4 + default: + return "SizeB1024(" + strconv.FormatInt(int64(i), 10) + ")" + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/speed.go b/vendor/github.com/vbauerster/mpb/v7/decor/speed.go new file mode 100644 index 000000000..634edabfd --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/speed.go @@ -0,0 +1,171 @@ +package decor + +import ( + "fmt" + "io" + "math" + "time" + + "github.com/VividCortex/ewma" +) + +// FmtAsSpeed adds "/s" to the end of the input formatter. To be +// used with SizeB1000 or SizeB1024 types, for example: +// +// fmt.Printf("%.1f", FmtAsSpeed(SizeB1024(2048))) +// +func FmtAsSpeed(input fmt.Formatter) fmt.Formatter { + return &speedFormatter{input} +} + +type speedFormatter struct { + fmt.Formatter +} + +func (self *speedFormatter) Format(st fmt.State, verb rune) { + self.Formatter.Format(st, verb) + io.WriteString(st, "/s") +} + +// EwmaSpeed exponential-weighted-moving-average based speed decorator. +// For this decorator to work correctly you have to measure each +// iteration's duration and pass it to the +// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment. +func EwmaSpeed(unit int, format string, age float64, wcc ...WC) Decorator { + var average ewma.MovingAverage + if age == 0 { + average = ewma.NewMovingAverage() + } else { + average = ewma.NewMovingAverage(age) + } + return MovingAverageSpeed(unit, format, NewThreadSafeMovingAverage(average), wcc...) +} + +// MovingAverageSpeed decorator relies on MovingAverage implementation +// to calculate its average. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `format` printf compatible verb for value, like "%f" or "%d" +// +// `average` MovingAverage implementation +// +// `wcc` optional WC config +// +// format examples: +// +// unit=UnitKiB, format="%.1f" output: "1.0MiB/s" +// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s" +// unit=UnitKB, format="%.1f" output: "1.0MB/s" +// unit=UnitKB, format="% .1f" output: "1.0 MB/s" +// +func MovingAverageSpeed(unit int, format string, average ewma.MovingAverage, wcc ...WC) Decorator { + if format == "" { + format = "%.0f" + } + d := &movingAverageSpeed{ + WC: initWC(wcc...), + average: average, + producer: chooseSpeedProducer(unit, format), + } + return d +} + +type movingAverageSpeed struct { + WC + producer func(float64) string + average ewma.MovingAverage + msg string +} + +func (d *movingAverageSpeed) Decor(s Statistics) string { + if !s.Completed { + var speed float64 + if v := d.average.Value(); v > 0 { + speed = 1 / v + } + d.msg = d.producer(speed * 1e9) + } + return d.FormatMsg(d.msg) +} + +func (d *movingAverageSpeed) EwmaUpdate(n int64, dur time.Duration) { + durPerByte := float64(dur) / float64(n) + if math.IsInf(durPerByte, 0) || math.IsNaN(durPerByte) { + return + } + d.average.Add(durPerByte) +} + +// AverageSpeed decorator with dynamic unit measure adjustment. It's +// a wrapper of NewAverageSpeed. +func AverageSpeed(unit int, format string, wcc ...WC) Decorator { + return NewAverageSpeed(unit, format, time.Now(), wcc...) +} + +// NewAverageSpeed decorator with dynamic unit measure adjustment and +// user provided start time. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `format` printf compatible verb for value, like "%f" or "%d" +// +// `startTime` start time +// +// `wcc` optional WC config +// +// format examples: +// +// unit=UnitKiB, format="%.1f" output: "1.0MiB/s" +// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s" +// unit=UnitKB, format="%.1f" output: "1.0MB/s" +// unit=UnitKB, format="% .1f" output: "1.0 MB/s" +// +func NewAverageSpeed(unit int, format string, startTime time.Time, wcc ...WC) Decorator { + if format == "" { + format = "%.0f" + } + d := &averageSpeed{ + WC: initWC(wcc...), + startTime: startTime, + producer: chooseSpeedProducer(unit, format), + } + return d +} + +type averageSpeed struct { + WC + startTime time.Time + producer func(float64) string + msg string +} + +func (d *averageSpeed) Decor(s Statistics) string { + if !s.Completed { + speed := float64(s.Current) / float64(time.Since(d.startTime)) + d.msg = d.producer(speed * 1e9) + } + + return d.FormatMsg(d.msg) +} + +func (d *averageSpeed) AverageAdjust(startTime time.Time) { + d.startTime = startTime +} + +func chooseSpeedProducer(unit int, format string) func(float64) string { + switch unit { + case UnitKiB: + return func(speed float64) string { + return fmt.Sprintf(format, FmtAsSpeed(SizeB1024(math.Round(speed)))) + } + case UnitKB: + return func(speed float64) string { + return fmt.Sprintf(format, FmtAsSpeed(SizeB1000(math.Round(speed)))) + } + default: + return func(speed float64) string { + return fmt.Sprintf(format, speed) + } + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/spinner.go b/vendor/github.com/vbauerster/mpb/v7/decor/spinner.go new file mode 100644 index 000000000..6871639db --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/decor/spinner.go @@ -0,0 +1,21 @@ +package decor + +var defaultSpinnerStyle = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} + +// Spinner returns spinner decorator. +// +// `frames` spinner frames, if nil or len==0, default is used +// +// `wcc` optional WC config +func Spinner(frames []string, wcc ...WC) Decorator { + if len(frames) == 0 { + frames = defaultSpinnerStyle + } + var count uint + f := func(s Statistics) string { + frame := frames[count%uint(len(frames))] + count++ + return frame + } + return Any(f, wcc...) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/doc.go b/vendor/github.com/vbauerster/mpb/v7/doc.go new file mode 100644 index 000000000..5ada71774 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/doc.go @@ -0,0 +1,2 @@ +// Package mpb is a library for rendering progress bars in terminal applications. +package mpb diff --git a/vendor/github.com/vbauerster/mpb/v7/go.mod b/vendor/github.com/vbauerster/mpb/v7/go.mod new file mode 100644 index 000000000..ea22e1eda --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/go.mod @@ -0,0 +1,10 @@ +module github.com/vbauerster/mpb/v7 + +require ( + github.com/VividCortex/ewma v1.2.0 + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d + github.com/mattn/go-runewidth v0.0.13 + golang.org/x/sys v0.0.0-20210603125802-9665404d3644 +) + +go 1.14 diff --git a/vendor/github.com/vbauerster/mpb/v7/go.sum b/vendor/github.com/vbauerster/mpb/v7/go.sum new file mode 100644 index 000000000..0b609b223 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/go.sum @@ -0,0 +1,10 @@ +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/vendor/github.com/vbauerster/mpb/v7/internal/percentage.go b/vendor/github.com/vbauerster/mpb/v7/internal/percentage.go new file mode 100644 index 000000000..a8ef8be12 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/internal/percentage.go @@ -0,0 +1,19 @@ +package internal + +import "math" + +// Percentage is a helper function, to calculate percentage. +func Percentage(total, current int64, width int) float64 { + if total <= 0 { + return 0 + } + if current >= total { + return float64(width) + } + return float64(int64(width)*current) / float64(total) +} + +// PercentageRound same as Percentage but with math.Round. +func PercentageRound(total, current int64, width int) float64 { + return math.Round(Percentage(total, current, width)) +} diff --git a/vendor/github.com/vbauerster/mpb/v7/internal/predicate.go b/vendor/github.com/vbauerster/mpb/v7/internal/predicate.go new file mode 100644 index 000000000..1e4dd24d9 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/internal/predicate.go @@ -0,0 +1,6 @@ +package internal + +// Predicate helper for internal use. +func Predicate(pick bool) func() bool { + return func() bool { return pick } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/internal/width.go b/vendor/github.com/vbauerster/mpb/v7/internal/width.go new file mode 100644 index 000000000..7677e404a --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/internal/width.go @@ -0,0 +1,10 @@ +package internal + +// CheckRequestedWidth checks that requested width doesn't overflow +// available width +func CheckRequestedWidth(requested, available int) int { + if requested < 1 || requested >= available { + return available + } + return requested +} diff --git a/vendor/github.com/vbauerster/mpb/v7/priority_queue.go b/vendor/github.com/vbauerster/mpb/v7/priority_queue.go new file mode 100644 index 000000000..29d9bd5a8 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/priority_queue.go @@ -0,0 +1,32 @@ +package mpb + +// A priorityQueue implements heap.Interface +type priorityQueue []*Bar + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].priority < pq[j].priority +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + s := *pq + bar := x.(*Bar) + bar.index = len(s) + s = append(s, bar) + *pq = s +} + +func (pq *priorityQueue) Pop() interface{} { + s := *pq + *pq = s[0 : len(s)-1] + bar := s[len(s)-1] + bar.index = -1 // for safety + return bar +} diff --git a/vendor/github.com/vbauerster/mpb/v7/progress.go b/vendor/github.com/vbauerster/mpb/v7/progress.go new file mode 100644 index 000000000..b2017f3f0 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/progress.go @@ -0,0 +1,412 @@ +package mpb + +import ( + "bytes" + "container/heap" + "context" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "os" + "sync" + "time" + + "github.com/vbauerster/mpb/v7/cwriter" + "github.com/vbauerster/mpb/v7/decor" +) + +const ( + // default RefreshRate + prr = 120 * time.Millisecond +) + +// Progress represents a container that renders one or more progress +// bars. +type Progress struct { + ctx context.Context + uwg *sync.WaitGroup + cwg *sync.WaitGroup + bwg *sync.WaitGroup + operateState chan func(*pState) + done chan struct{} + refreshCh chan time.Time + once sync.Once + dlogger *log.Logger +} + +// pState holds bars in its priorityQueue. It gets passed to +// *Progress.serve(...) monitor goroutine. +type pState struct { + bHeap priorityQueue + heapUpdated bool + pMatrix map[int][]chan int + aMatrix map[int][]chan int + barShutdownQueue []*Bar + + // following are provided/overrided by user + idCount int + reqWidth int + popCompleted bool + outputDiscarded bool + rr time.Duration + uwg *sync.WaitGroup + externalRefresh <-chan interface{} + renderDelay <-chan struct{} + shutdownNotifier chan struct{} + parkedBars map[*Bar]*Bar + output io.Writer + debugOut io.Writer +} + +// New creates new Progress container instance. It's not possible to +// reuse instance after *Progress.Wait() method has been called. +func New(options ...ContainerOption) *Progress { + return NewWithContext(context.Background(), options...) +} + +// NewWithContext creates new Progress container instance with provided +// context. It's not possible to reuse instance after *Progress.Wait() +// method has been called. +func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress { + s := &pState{ + bHeap: priorityQueue{}, + rr: prr, + parkedBars: make(map[*Bar]*Bar), + output: os.Stdout, + debugOut: ioutil.Discard, + } + + for _, opt := range options { + if opt != nil { + opt(s) + } + } + + p := &Progress{ + ctx: ctx, + uwg: s.uwg, + cwg: new(sync.WaitGroup), + bwg: new(sync.WaitGroup), + operateState: make(chan func(*pState)), + done: make(chan struct{}), + dlogger: log.New(s.debugOut, "[mpb] ", log.Lshortfile), + } + + p.cwg.Add(1) + go p.serve(s, cwriter.New(s.output)) + return p +} + +// AddBar creates a bar with default bar filler. Different filler can +// be chosen and applied via `*Progress.Add(...) *Bar` method. +func (p *Progress) AddBar(total int64, options ...BarOption) *Bar { + return p.Add(total, NewBarFiller(BarStyle()), options...) +} + +// AddSpinner creates a bar with default spinner filler. Different +// filler can be chosen and applied via `*Progress.Add(...) *Bar` +// method. +func (p *Progress) AddSpinner(total int64, options ...BarOption) *Bar { + return p.Add(total, NewBarFiller(SpinnerStyle()), options...) +} + +// Add creates a bar which renders itself by provided filler. +// If `total <= 0` trigger complete event is disabled until reset with *bar.SetTotal(int64, bool). +// Panics if *Progress instance is done, i.e. called after *Progress.Wait(). +func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar { + if filler == nil { + filler = BarFillerFunc(func(io.Writer, int, decor.Statistics) {}) + } + p.bwg.Add(1) + result := make(chan *Bar) + select { + case p.operateState <- func(ps *pState) { + bs := ps.makeBarState(total, filler, options...) + bar := newBar(p, bs) + if bs.runningBar != nil { + bs.runningBar.noPop = true + ps.parkedBars[bs.runningBar] = bar + } else { + heap.Push(&ps.bHeap, bar) + ps.heapUpdated = true + } + ps.idCount++ + result <- bar + }: + bar := <-result + bar.subscribeDecorators() + return bar + case <-p.done: + p.bwg.Done() + panic(fmt.Sprintf("%T instance can't be reused after it's done!", p)) + } +} + +func (p *Progress) dropBar(b *Bar) { + select { + case p.operateState <- func(s *pState) { + if b.index < 0 { + return + } + heap.Remove(&s.bHeap, b.index) + s.heapUpdated = true + }: + case <-p.done: + } +} + +func (p *Progress) setBarPriority(b *Bar, priority int) { + select { + case p.operateState <- func(s *pState) { + if b.index < 0 { + return + } + b.priority = priority + heap.Fix(&s.bHeap, b.index) + }: + case <-p.done: + } +} + +// UpdateBarPriority same as *Bar.SetPriority(int). +func (p *Progress) UpdateBarPriority(b *Bar, priority int) { + p.setBarPriority(b, priority) +} + +// BarCount returns bars count. +func (p *Progress) BarCount() int { + result := make(chan int, 1) + select { + case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }: + return <-result + case <-p.done: + return 0 + } +} + +// Wait waits for all bars to complete and finally shutdowns container. +// After this method has been called, there is no way to reuse *Progress +// instance. +func (p *Progress) Wait() { + if p.uwg != nil { + // wait for user wg + p.uwg.Wait() + } + + // wait for bars to quit, if any + p.bwg.Wait() + + p.once.Do(p.shutdown) + + // wait for container to quit + p.cwg.Wait() +} + +func (p *Progress) shutdown() { + close(p.done) +} + +func (p *Progress) serve(s *pState, cw *cwriter.Writer) { + defer p.cwg.Done() + + p.refreshCh = s.newTicker(p.done) + + for { + select { + case op := <-p.operateState: + op(s) + case <-p.refreshCh: + if err := s.render(cw); err != nil { + p.dlogger.Println(err) + } + case <-s.shutdownNotifier: + if s.heapUpdated { + if err := s.render(cw); err != nil { + p.dlogger.Println(err) + } + } + return + } + } +} + +func (s *pState) newTicker(done <-chan struct{}) chan time.Time { + ch := make(chan time.Time) + if s.shutdownNotifier == nil { + s.shutdownNotifier = make(chan struct{}) + } + go func() { + if s.renderDelay != nil { + <-s.renderDelay + } + var internalRefresh <-chan time.Time + if !s.outputDiscarded { + if s.externalRefresh == nil { + ticker := time.NewTicker(s.rr) + defer ticker.Stop() + internalRefresh = ticker.C + } + } else { + s.externalRefresh = nil + } + for { + select { + case t := <-internalRefresh: + ch <- t + case x := <-s.externalRefresh: + if t, ok := x.(time.Time); ok { + ch <- t + } else { + ch <- time.Now() + } + case <-done: + close(s.shutdownNotifier) + return + } + } + }() + return ch +} + +func (s *pState) render(cw *cwriter.Writer) error { + if s.heapUpdated { + s.updateSyncMatrix() + s.heapUpdated = false + } + syncWidth(s.pMatrix) + syncWidth(s.aMatrix) + + tw, err := cw.GetWidth() + if err != nil { + tw = s.reqWidth + } + for i := 0; i < s.bHeap.Len(); i++ { + bar := s.bHeap[i] + go bar.render(tw) + } + + return s.flush(cw) +} + +func (s *pState) flush(cw *cwriter.Writer) error { + var lineCount int + bm := make(map[*Bar]struct{}, s.bHeap.Len()) + for s.bHeap.Len() > 0 { + b := heap.Pop(&s.bHeap).(*Bar) + cw.ReadFrom(<-b.frameCh) + if b.toShutdown { + if b.recoveredPanic != nil { + s.barShutdownQueue = append(s.barShutdownQueue, b) + b.toShutdown = false + } else { + // shutdown at next flush + // this ensures no bar ends up with less than 100% rendered + defer func() { + s.barShutdownQueue = append(s.barShutdownQueue, b) + }() + } + } + lineCount += b.extendedLines + 1 + bm[b] = struct{}{} + } + + for _, b := range s.barShutdownQueue { + if parkedBar := s.parkedBars[b]; parkedBar != nil { + parkedBar.priority = b.priority + heap.Push(&s.bHeap, parkedBar) + delete(s.parkedBars, b) + b.toDrop = true + } + if s.popCompleted && !b.noPop { + lineCount -= b.extendedLines + 1 + b.toDrop = true + } + if b.toDrop { + delete(bm, b) + s.heapUpdated = true + } + b.cancel() + } + s.barShutdownQueue = s.barShutdownQueue[0:0] + + for b := range bm { + heap.Push(&s.bHeap, b) + } + + return cw.Flush(lineCount) +} + +func (s *pState) updateSyncMatrix() { + s.pMatrix = make(map[int][]chan int) + s.aMatrix = make(map[int][]chan int) + for i := 0; i < s.bHeap.Len(); i++ { + bar := s.bHeap[i] + table := bar.wSyncTable() + pRow, aRow := table[0], table[1] + + for i, ch := range pRow { + s.pMatrix[i] = append(s.pMatrix[i], ch) + } + + for i, ch := range aRow { + s.aMatrix[i] = append(s.aMatrix[i], ch) + } + } +} + +func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState { + bs := &bState{ + id: s.idCount, + priority: s.idCount, + reqWidth: s.reqWidth, + total: total, + filler: filler, + extender: func(r io.Reader, _ int, _ decor.Statistics) (io.Reader, int) { return r, 0 }, + debugOut: s.debugOut, + } + + if total > 0 { + bs.triggerComplete = true + } + + for _, opt := range options { + if opt != nil { + opt(bs) + } + } + + if bs.middleware != nil { + bs.filler = bs.middleware(filler) + bs.middleware = nil + } + + if s.popCompleted && !bs.noPop { + bs.priority = -(math.MaxInt32 - s.idCount) + } + + bs.bufP = bytes.NewBuffer(make([]byte, 0, 128)) + bs.bufB = bytes.NewBuffer(make([]byte, 0, 256)) + bs.bufA = bytes.NewBuffer(make([]byte, 0, 128)) + + return bs +} + +func syncWidth(matrix map[int][]chan int) { + for _, column := range matrix { + go maxWidthDistributor(column) + } +} + +var maxWidthDistributor = func(column []chan int) { + var maxWidth int + for _, ch := range column { + if w := <-ch; w > maxWidth { + maxWidth = w + } + } + for _, ch := range column { + ch <- maxWidth + } +} diff --git a/vendor/github.com/vbauerster/mpb/v7/proxyreader.go b/vendor/github.com/vbauerster/mpb/v7/proxyreader.go new file mode 100644 index 000000000..316f438d7 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/v7/proxyreader.go @@ -0,0 +1,90 @@ +package mpb + +import ( + "io" + "io/ioutil" + "time" +) + +type proxyReader struct { + io.ReadCloser + bar *Bar +} + +func (x *proxyReader) Read(p []byte) (int, error) { + n, err := x.ReadCloser.Read(p) + x.bar.IncrBy(n) + if err == io.EOF { + go x.bar.SetTotal(0, true) + } + return n, err +} + +type proxyWriterTo struct { + io.ReadCloser // *proxyReader + wt io.WriterTo + bar *Bar +} + +func (x *proxyWriterTo) WriteTo(w io.Writer) (int64, error) { + n, err := x.wt.WriteTo(w) + x.bar.IncrInt64(n) + if err == io.EOF { + go x.bar.SetTotal(0, true) + } + return n, err +} + +type ewmaProxyReader struct { + io.ReadCloser // *proxyReader + bar *Bar + iT time.Time +} + +func (x *ewmaProxyReader) Read(p []byte) (int, error) { + n, err := x.ReadCloser.Read(p) + if n > 0 { + x.bar.DecoratorEwmaUpdate(time.Since(x.iT)) + x.iT = time.Now() + } + return n, err +} + +type ewmaProxyWriterTo struct { + io.ReadCloser // *ewmaProxyReader + wt io.WriterTo // *proxyWriterTo + bar *Bar + iT time.Time +} + +func (x *ewmaProxyWriterTo) WriteTo(w io.Writer) (int64, error) { + n, err := x.wt.WriteTo(w) + if n > 0 { + x.bar.DecoratorEwmaUpdate(time.Since(x.iT)) + x.iT = time.Now() + } + return n, err +} + +func newProxyReader(r io.Reader, bar *Bar) io.ReadCloser { + rc := toReadCloser(r) + rc = &proxyReader{rc, bar} + + if wt, isWriterTo := r.(io.WriterTo); bar.hasEwmaDecorators { + now := time.Now() + rc = &ewmaProxyReader{rc, bar, now} + if isWriterTo { + rc = &ewmaProxyWriterTo{rc, wt, bar, now} + } + } else if isWriterTo { + rc = &proxyWriterTo{rc, wt, bar} + } + return rc +} + +func toReadCloser(r io.Reader) io.ReadCloser { + if rc, ok := r.(io.ReadCloser); ok { + return rc + } + return ioutil.NopCloser(r) +} |