diff options
122 files changed, 4666 insertions, 2898 deletions
@@ -51,6 +51,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func GetPodStats(name: string) string, ContainerStats](#GetPodStats) +[func GetPodsByContext(all: bool, latest: bool, args: []string) []string](#GetPodsByContext) + [func GetVersion() string, string, string, string, string, int](#GetVersion) [func GetVolumes(args: []string, all: bool) Volume](#GetVolumes) @@ -59,6 +61,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func ImageExists(name: string) int](#ImageExists) +[func ImageSave(options: ImageSaveOptions) MoreResponse](#ImageSave) + [func ImagesPrune(all: bool) []string](#ImagesPrune) [func ImportImage(source: string, reference: string, message: string, changes: []string, delete: bool) string](#ImportImage) @@ -107,7 +111,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func RestartPod(name: string) string](#RestartPod) -[func SearchImages(query: string, limit: int, tlsVerify: ?bool, filter: ImageSearchFilter) ImageSearchResult](#SearchImages) +[func SearchImages(query: string, limit: , tlsVerify: , filter: ImageSearchFilter) ImageSearchResult](#SearchImages) [func SendFile(type: string, length: int) string](#SendFile) @@ -163,6 +167,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [type ImageHistory](#ImageHistory) +[type ImageSaveOptions](#ImageSaveOptions) + [type ImageSearchFilter](#ImageSearchFilter) [type ImageSearchResult](#ImageSearchResult) @@ -525,6 +531,11 @@ $ varlink call unix:/run/podman/io.podman/io.podman.GetPodStats '{"name": "7f62b "pod": "7f62b508b6f12b11d8fe02e0db4de6b9e43a7d7699b33a4fc0d574f6e82b4ebd" } ~~~ +### <a name="GetPodsByContext"></a>func GetPodsByContext +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method GetPodsByContext(all: [bool](https://godoc.org/builtin#bool), latest: [bool](https://godoc.org/builtin#bool), args: [[]string](#[]string)) [[]string](#[]string)</div> + ### <a name="GetVersion"></a>func GetVersion <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -556,6 +567,11 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.ImageExists '{"name": "im "exists": 1 } ~~~ +### <a name="ImageSave"></a>func ImageSave +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ImageSave(options: [ImageSaveOptions](#ImageSaveOptions)) [MoreResponse](#MoreResponse)</div> + ### <a name="ImagesPrune"></a>func ImagesPrune <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -847,7 +863,7 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.RestartPod '{"name": "135 ### <a name="SearchImages"></a>func SearchImages <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> -method SearchImages(query: [string](https://godoc.org/builtin#string), limit: [](#), tlsVerify: [](#)) [ImageSearchResult](#ImageSearchResult)</div> +method SearchImages(query: [string](https://godoc.org/builtin#string), limit: [](#), tlsVerify: [](#), filter: [ImageSearchFilter](#ImageSearchFilter)) [ImageSearchResult](#ImageSearchResult)</div> SearchImages searches available registries for images that contain the contents of "query" in their name. If "limit" is given, limits the amount of search results per registry. @@ -1410,13 +1426,30 @@ tags [[]string](#[]string) size [int](https://godoc.org/builtin#int) comment [string](https://godoc.org/builtin#string) +### <a name="ImageSaveOptions"></a>type ImageSaveOptions + + + +name [string](https://godoc.org/builtin#string) + +format [string](https://godoc.org/builtin#string) + +output [string](https://godoc.org/builtin#string) + +outputType [string](https://godoc.org/builtin#string) + +moreTags [[]string](#[]string) + +quiet [bool](https://godoc.org/builtin#bool) + +compress [bool](https://godoc.org/builtin#bool) ### <a name="ImageSearchFilter"></a>type ImageSearchFilter -Represents a filter for SearchImages -is_official [bool](https://godoc.org/builtin#bool) -is_automated [bool](https://godoc.org/builtin#bool) +is_official [](#) + +is_automated [](#) star_count [int](https://godoc.org/builtin#int) ### <a name="ImageSearchResult"></a>type ImageSearchResult diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index ca788529c..a9032202f 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -548,3 +548,7 @@ type SystemPruneValues struct { Force bool Volume bool } + +type SystemRenumberValues struct { + PodmanCommand +} diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go index 27ac342ba..0acb9c398 100644 --- a/cmd/podman/commands.go +++ b/cmd/podman/commands.go @@ -15,10 +15,8 @@ func getMainCommands() []*cobra.Command { _diffCommand, _execCommand, generateCommand.Command, - podCommand.Command, _containerKubeCommand, _psCommand, - _loadCommand, _loginCommand, _logoutCommand, _logsCommand, @@ -30,7 +28,6 @@ func getMainCommands() []*cobra.Command { _restoreCommand, _rmCommand, _runCommand, - _saveCommand, _searchCommand, _signCommand, _startCommand, @@ -53,7 +50,6 @@ func getMainCommands() []*cobra.Command { func getImageSubCommands() []*cobra.Command { return []*cobra.Command{ _loadCommand, - _saveCommand, _signCommand, } } @@ -102,7 +98,6 @@ func getPodSubCommands() []*cobra.Command { _podPauseCommand, _podPsCommand, _podRestartCommand, - _podRmCommand, _podStartCommand, _podStatsCommand, _podStopCommand, @@ -136,5 +131,6 @@ func getTrustSubCommands() []*cobra.Command { func getSystemSubCommands() []*cobra.Command { return []*cobra.Command{ _pruneSystemCommand, + _renumberCommand, } } diff --git a/cmd/podman/image.go b/cmd/podman/image.go index 4f9c7cd6a..14053cb0d 100644 --- a/cmd/podman/image.go +++ b/cmd/podman/image.go @@ -24,10 +24,12 @@ var imageSubCommands = []*cobra.Command{ _imagesCommand, _importCommand, _inspectCommand, + _loadCommand, _pruneImagesCommand, _pullCommand, _pushCommand, _rmiCommand, + _saveCommand, _tagCommand, } diff --git a/cmd/podman/libpodruntime/runtime.go b/cmd/podman/libpodruntime/runtime.go index 0b9568b8d..880b281bd 100644 --- a/cmd/podman/libpodruntime/runtime.go +++ b/cmd/podman/libpodruntime/runtime.go @@ -8,8 +8,17 @@ import ( "github.com/pkg/errors" ) +// GetRuntimeRenumber gets a libpod runtime that will perform a lock renumber +func GetRuntimeRenumber(c *cliconfig.PodmanCommand) (*libpod.Runtime, error) { + return getRuntime(c, true) +} + // GetRuntime generates a new libpod runtime configured by command line options func GetRuntime(c *cliconfig.PodmanCommand) (*libpod.Runtime, error) { + return getRuntime(c, false) +} + +func getRuntime(c *cliconfig.PodmanCommand, renumber bool) (*libpod.Runtime, error) { options := []libpod.RuntimeOption{} storageOpts, volumePath, err := util.GetDefaultStoreOptions() diff --git a/cmd/podman/load.go b/cmd/podman/load.go index 34a51cd0d..514c9f1e9 100644 --- a/cmd/podman/load.go +++ b/cmd/podman/load.go @@ -6,12 +6,8 @@ import ( "io/ioutil" "os" - "github.com/containers/image/directory" - dockerarchive "github.com/containers/image/docker/archive" - ociarchive "github.com/containers/image/oci/archive" "github.com/containers/libpod/cmd/podman/cliconfig" - "github.com/containers/libpod/cmd/podman/libpodruntime" - "github.com/containers/libpod/libpod/image" + "github.com/containers/libpod/libpod/adapter" "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -56,14 +52,16 @@ func loadCmd(c *cliconfig.LoadValues) error { return errors.New("too many arguments. Requires exactly 1") } - runtime, err := libpodruntime.GetRuntime(&c.PodmanCommand) + runtime, err := adapter.GetRuntime(&c.PodmanCommand) if err != nil { return errors.Wrapf(err, "could not get runtime") } defer runtime.Shutdown(false) input := c.Input - + if runtime.Remote && len(input) == 0 { + return errors.New("the remote client requires you to load via -i and a tarball") + } if input == "/dev/stdin" { fi, err := os.Stdin.Stat() if err != nil { @@ -96,46 +94,10 @@ func loadCmd(c *cliconfig.LoadValues) error { return err } - var writer io.Writer - if !c.Quiet { - writer = os.Stderr - } - - ctx := getContext() - - var newImages []*image.Image - src, err := dockerarchive.ParseReference(input) // FIXME? We should add dockerarchive.NewReference() - if err == nil { - newImages, err = runtime.ImageRuntime().LoadFromArchiveReference(ctx, src, c.SignaturePolicy, writer) - } + names, err := runtime.LoadImage(getContext(), imageName, c) if err != nil { - // generate full src name with specified image:tag - src, err := ociarchive.NewReference(input, imageName) // imageName may be "" - if err == nil { - newImages, err = runtime.ImageRuntime().LoadFromArchiveReference(ctx, src, c.SignaturePolicy, writer) - } - if err != nil { - src, err := directory.NewReference(input) - if err == nil { - newImages, err = runtime.ImageRuntime().LoadFromArchiveReference(ctx, src, c.SignaturePolicy, writer) - } - if err != nil { - return errors.Wrapf(err, "error pulling %q", input) - } - } + return err } - fmt.Println("Loaded image(s): " + getImageNames(newImages)) + fmt.Println("Loaded image(s): " + names) return nil } - -func getImageNames(images []*image.Image) string { - var names string - for i := range images { - if i == 0 { - names = images[i].InputName - } else { - names += ", " + images[i].InputName - } - } - return names -} diff --git a/cmd/podman/main.go b/cmd/podman/main.go index ecb72f58b..990e55a8c 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -45,9 +45,12 @@ var mainCommands = []*cobra.Command{ _infoCommand, _inspectCommand, _killCommand, + _loadCommand, + podCommand.Command, _pullCommand, _pushCommand, _rmiCommand, + _saveCommand, _tagCommand, _versionCommand, imageCommand.Command, diff --git a/cmd/podman/pod.go b/cmd/podman/pod.go index e988875ab..cf87730d7 100644 --- a/cmd/podman/pod.go +++ b/cmd/podman/pod.go @@ -18,7 +18,13 @@ var podCommand = cliconfig.PodmanCommand{ }, } +//podSubCommands are implemented both in local and remote clients +var podSubCommands = []*cobra.Command{ + _podRmCommand, +} + func init() { + podCommand.AddCommand(podSubCommands...) podCommand.AddCommand(getPodSubCommands()...) podCommand.SetUsageTemplate(UsageTemplate()) } diff --git a/cmd/podman/pod_rm.go b/cmd/podman/pod_rm.go index 54cee2a50..b615f88c9 100644 --- a/cmd/podman/pod_rm.go +++ b/cmd/podman/pod_rm.go @@ -2,9 +2,9 @@ package main import ( "fmt" + "github.com/containers/libpod/libpod/adapter" "github.com/containers/libpod/cmd/podman/cliconfig" - "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -42,36 +42,30 @@ func init() { } -// saveCmd saves the image to either docker-archive or oci +// podRmCmd deletes pods func podRmCmd(c *cliconfig.PodRmValues) error { if err := checkMutuallyExclusiveFlags(&c.PodmanCommand); err != nil { return err } - - runtime, err := libpodruntime.GetRuntime(&c.PodmanCommand) + runtime, err := adapter.GetRuntime(&c.PodmanCommand) if err != nil { return errors.Wrapf(err, "could not get runtime") } defer runtime.Shutdown(false) + podRmIds, podRmErrors := runtime.RemovePods(getContext(), c) + for _, p := range podRmIds { + fmt.Println(p) + } + if len(podRmErrors) == 0 { + return nil + } + // Grab the last error + lastError := podRmErrors[len(podRmErrors)-1] + // Remove the last error from the error slice + podRmErrors = podRmErrors[:len(podRmErrors)-1] - ctx := getContext() - force := c.Force - - // getPodsFromContext returns an error when a requested pod - // isn't found. The only fatal error scenerio is when there are no pods - // in which case the following loop will be skipped. - pods, lastError := getPodsFromContext(&c.PodmanCommand, runtime) - - for _, pod := range pods { - err = runtime.RemovePod(ctx, pod, force, force) - if err != nil { - if lastError != nil { - logrus.Errorf("%q", lastError) - } - lastError = errors.Wrapf(err, "failed to delete pod %v", pod.ID()) - } else { - fmt.Println(pod.ID()) - } + for _, err := range podRmErrors { + logrus.Errorf("%q", err) } return lastError } diff --git a/cmd/podman/save.go b/cmd/podman/save.go index ff4a22453..ba5209f34 100644 --- a/cmd/podman/save.go +++ b/cmd/podman/save.go @@ -1,21 +1,10 @@ package main import ( - "fmt" - "io" "os" - "strings" - "github.com/containers/image/directory" - dockerarchive "github.com/containers/image/docker/archive" - "github.com/containers/image/docker/reference" - "github.com/containers/image/manifest" - ociarchive "github.com/containers/image/oci/archive" - "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/cliconfig" - "github.com/containers/libpod/cmd/podman/libpodruntime" - libpodImage "github.com/containers/libpod/libpod/image" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/containers/libpod/libpod/adapter" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -52,7 +41,7 @@ func init() { saveCommand.SetUsageTemplate(UsageTemplate()) flags := saveCommand.Flags() flags.BoolVar(&saveCommand.Compress, "compress", false, "Compress tarball image layers when saving to a directory using the 'dir' transport. (default is same compression type as source)") - flags.StringVar(&saveCommand.Format, "format", "", "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-dir (directory with v2s2 manifest type)") + flags.StringVar(&saveCommand.Format, "format", "docker-archive", "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-dir (directory with v2s2 manifest type)") flags.StringVarP(&saveCommand.Output, "output", "o", "/dev/stdout", "Write to a file, default is STDOUT") flags.BoolVarP(&saveCommand.Quiet, "quiet", "q", false, "Suppress the output") } @@ -64,7 +53,7 @@ func saveCmd(c *cliconfig.SaveValues) error { return errors.Errorf("need at least 1 argument") } - runtime, err := libpodruntime.GetRuntime(&c.PodmanCommand) + runtime, err := adapter.GetRuntime(&c.PodmanCommand) if err != nil { return errors.Wrapf(err, "could not create runtime") } @@ -74,11 +63,6 @@ func saveCmd(c *cliconfig.SaveValues) error { return errors.Errorf("--compress can only be set when --format is either 'oci-dir' or 'docker-dir'") } - var writer io.Writer - if !c.Quiet { - writer = os.Stderr - } - output := c.Output if output == "/dev/stdout" { fi := os.Stdout @@ -89,87 +73,5 @@ func saveCmd(c *cliconfig.SaveValues) error { if err := validateFileName(output); err != nil { return err } - - source := args[0] - newImage, err := runtime.ImageRuntime().NewFromLocal(source) - if err != nil { - return err - } - - var destRef types.ImageReference - var manifestType string - switch c.Format { - case "oci-archive": - destImageName := imageNameForSaveDestination(newImage, source) - destRef, err = ociarchive.NewReference(output, destImageName) // destImageName may be "" - if err != nil { - return errors.Wrapf(err, "error getting OCI archive ImageReference for (%q, %q)", output, destImageName) - } - case "oci-dir": - destRef, err = directory.NewReference(output) - if err != nil { - return errors.Wrapf(err, "error getting directory ImageReference for %q", output) - } - manifestType = imgspecv1.MediaTypeImageManifest - case "docker-dir": - destRef, err = directory.NewReference(output) - if err != nil { - return errors.Wrapf(err, "error getting directory ImageReference for %q", output) - } - manifestType = manifest.DockerV2Schema2MediaType - case "docker-archive", "": - dst := output - destImageName := imageNameForSaveDestination(newImage, source) - if destImageName != "" { - dst = fmt.Sprintf("%s:%s", dst, destImageName) - } - destRef, err = dockerarchive.ParseReference(dst) // FIXME? Add dockerarchive.NewReference - if err != nil { - return errors.Wrapf(err, "error getting Docker archive ImageReference for %q", dst) - } - default: - return errors.Errorf("unknown format option %q", c.String("format")) - } - - // supports saving multiple tags to the same tar archive - var additionaltags []reference.NamedTagged - if len(args) > 1 { - additionaltags, err = libpodImage.GetAdditionalTags(args[1:]) - if err != nil { - return err - } - } - if err := newImage.PushImageToReference(getContext(), destRef, manifestType, "", "", writer, c.Bool("compress"), libpodImage.SigningOptions{}, &libpodImage.DockerRegistryOptions{}, additionaltags); err != nil { - if err2 := os.Remove(output); err2 != nil { - logrus.Errorf("error deleting %q: %v", output, err) - } - return errors.Wrapf(err, "unable to save %q", args) - } - - return nil -} - -// imageNameForSaveDestination returns a Docker-like reference appropriate for saving img, -// which the user referred to as imgUserInput; or an empty string, if there is no appropriate -// reference. -func imageNameForSaveDestination(img *libpodImage.Image, imgUserInput string) string { - if strings.Contains(img.ID(), imgUserInput) { - return "" - } - - prepend := "" - localRegistryPrefix := fmt.Sprintf("%s/", libpodImage.DefaultLocalRegistry) - if !strings.HasPrefix(imgUserInput, localRegistryPrefix) { - // we need to check if localhost was added to the image name in NewFromLocal - for _, name := range img.Names() { - // If the user is saving an image in the localhost registry, getLocalImage need - // a name that matches the format localhost/<tag1>:<tag2> or localhost/<tag>:latest to correctly - // set up the manifest and save. - if strings.HasPrefix(name, localRegistryPrefix) && (strings.HasSuffix(name, imgUserInput) || strings.HasSuffix(name, fmt.Sprintf("%s:latest", imgUserInput))) { - prepend = localRegistryPrefix - break - } - } - } - return fmt.Sprintf("%s%s", prepend, imgUserInput) + return runtime.SaveImage(getContext(), c) } diff --git a/cmd/podman/system_renumber.go b/cmd/podman/system_renumber.go new file mode 100644 index 000000000..c8ce628b1 --- /dev/null +++ b/cmd/podman/system_renumber.go @@ -0,0 +1,49 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/cliconfig" + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + renumberCommand cliconfig.SystemRenumberValues + renumberDescription = ` + podman system renumber + + Migrate lock numbers to handle a change in maximum number of locks. + Mandatory after the number of locks in libpod.conf is changed. +` + + _renumberCommand = &cobra.Command{ + Use: "renumber", + Short: "Migrate lock numbers", + Long: renumberDescription, + RunE: func(cmd *cobra.Command, args []string) error { + renumberCommand.InputArgs = args + renumberCommand.GlobalFlags = MainGlobalOpts + return renumberCmd(&renumberCommand) + }, + } +) + +func init() { + renumberCommand.Command = _renumberCommand + renumberCommand.SetUsageTemplate(UsageTemplate()) +} + +func renumberCmd(c *cliconfig.SystemRenumberValues) error { + // We need to pass one extra option to NewRuntime. + // This will inform the OCI runtime to start a renumber. + // That's controlled by the last argument to GetRuntime. + r, err := libpodruntime.GetRuntimeRenumber(&c.PodmanCommand) + if err != nil { + return errors.Wrapf(err, "error renumbering locks") + } + if err := r.Shutdown(false); err != nil { + return err + } + + return nil +} diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index c53a5454a..09d178760 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -26,6 +26,16 @@ type ContainerChanges ( deleted: []string ) +type ImageSaveOptions ( + name: string, + format: string, + output: string, + outputType: string, + moreTags: []string, + quiet: bool, + compress: bool +) + type VolumeCreateOpts ( volumeName: string, driver: string, @@ -1090,6 +1100,13 @@ method GetVolumes(args: []string, all: bool) -> (volumes: []Volume) # VolumesPrune removes unused volumes on the host method VolumesPrune() -> (prunedNames: []string, prunedErrors: []string) +method ImageSave(options: ImageSaveOptions) -> (reply: MoreResponse) + + +method GetPodsByContext(all: bool, latest: bool, args: []string) -> (pods: []string) + +method LoadImage(name: string, inputFile: string, quiet: bool, deleteFile: bool) -> (reply: MoreResponse) + # ImageNotFound means the image could not be found by the provided name or ID in local storage. error ImageNotFound (id: string) diff --git a/docs/podman-system-renumber.1.md b/docs/podman-system-renumber.1.md new file mode 100644 index 000000000..a88640d63 --- /dev/null +++ b/docs/podman-system-renumber.1.md @@ -0,0 +1,29 @@ +% podman-system-renumber(1) podman + +## NAME +podman\-system\-renumber - Renumber container locks + +## SYNOPSIS +** podman system renumber** + +## DESCRIPTION +** podman system renumber** renumbers locks used by containers and pods. + +Each Podman container and pod is allocated a lock at creation time, up to a maximum number controlled by the **num_locks** parameter in **libpod.conf**. + +When all available locks are exhausted, no further containers and pods can be created until some existing containers and pods are removed. This can be avoided by increasing the number of locks available via modifying **libpod.conf** and subsequently running **podman system renumber** to prepare the new locks (and reallocate lock numbers to fit the new struct). + +**podman system renumber** must be called after any changes to **num_locks** - failure to do so will result in errors starting Podman as the number of locks available conflicts with the configured number of locks. + +**podman system renumber** can also be used to migrate 1.0 and earlier versions of Podman, which used a different locking scheme, to the new locking model. It is not strictly required to do this, but it is highly recommended to do so as deadlocks can occur otherwise. + +If possible, avoid calling **podman system renumber** while there are other Podman processes running. + +## SYNOPSIS +**podman system renumber** + +## SEE ALSO +`podman(1)`, `libpod.conf(5)` + +# HISTORY +February 2019, Originally compiled by Matt Heon (mheon at redhat dot com) diff --git a/libpod/adapter/pods.go b/libpod/adapter/pods.go new file mode 100644 index 000000000..59642c42e --- /dev/null +++ b/libpod/adapter/pods.go @@ -0,0 +1,38 @@ +// +build !remoteclient + +package adapter + +import ( + "context" + "github.com/containers/libpod/libpod/adapter/shortcuts" + + "github.com/containers/libpod/cmd/podman/cliconfig" + "github.com/containers/libpod/libpod" +) + +// Pod ... +type Pod struct { + *libpod.Pod +} + +// RemovePods ... +func (r *LocalRuntime) RemovePods(ctx context.Context, cli *cliconfig.PodRmValues) ([]string, []error) { + var ( + errs []error + podids []string + ) + pods, err := shortcuts.GetPodsByContext(cli.All, cli.Latest, cli.InputArgs, r.Runtime) + if err != nil { + errs = append(errs, err) + return nil, errs + } + + for _, p := range pods { + if err := r.RemovePod(ctx, p, cli.Force, cli.Force); err != nil { + errs = append(errs, err) + } else { + podids = append(podids, p.ID()) + } + } + return podids, errs +} diff --git a/libpod/adapter/pods_remote.go b/libpod/adapter/pods_remote.go new file mode 100644 index 000000000..3fb147f48 --- /dev/null +++ b/libpod/adapter/pods_remote.go @@ -0,0 +1,44 @@ +// +build remoteclient + +package adapter + +import ( + "context" + + "github.com/containers/libpod/cmd/podman/cliconfig" + "github.com/containers/libpod/cmd/podman/varlink" + "github.com/containers/libpod/libpod" +) + +// Pod ... +type Pod struct { + remotepod +} + +type remotepod struct { + config *libpod.PodConfig + state *libpod.PodInspectState + Runtime *LocalRuntime +} + +func (r *LocalRuntime) RemovePods(ctx context.Context, cli *cliconfig.PodRmValues) ([]string, []error) { + var ( + rmErrs []error + rmPods []string + ) + podIDs, err := iopodman.GetPodsByContext().Call(r.Conn, cli.All, cli.Latest, cli.InputArgs) + if err != nil { + rmErrs = append(rmErrs, err) + return nil, rmErrs + } + + for _, p := range podIDs { + reply, err := iopodman.RemovePod().Call(r.Conn, p, cli.Force) + if err != nil { + rmErrs = append(rmErrs, err) + } else { + rmPods = append(rmPods, reply) + } + } + return rmPods, rmErrs +} diff --git a/libpod/adapter/runtime.go b/libpod/adapter/runtime.go index 02ef9af07..4f5b98dbb 100644 --- a/libpod/adapter/runtime.go +++ b/libpod/adapter/runtime.go @@ -310,3 +310,26 @@ func (r *LocalRuntime) Build(ctx context.Context, c *cliconfig.BuildValues, opti func (r *LocalRuntime) PruneVolumes(ctx context.Context) ([]string, []error) { return r.Runtime.PruneVolumes(ctx) } + +// SaveImage is a wrapper function for saving an image to the local filesystem +func (r *LocalRuntime) SaveImage(ctx context.Context, c *cliconfig.SaveValues) error { + source := c.InputArgs[0] + additionalTags := c.InputArgs[1:] + + newImage, err := r.Runtime.ImageRuntime().NewFromLocal(source) + if err != nil { + return err + } + return newImage.Save(ctx, source, c.Format, c.Output, additionalTags, c.Quiet, c.Compress) +} + +// LoadImage is a wrapper function for libpod PruneVolumes +func (r *LocalRuntime) LoadImage(ctx context.Context, name string, cli *cliconfig.LoadValues) (string, error) { + var ( + writer io.Writer + ) + if !cli.Quiet { + writer = os.Stderr + } + return r.Runtime.LoadImage(ctx, name, cli.Input, writer, cli.SignaturePolicy) +} diff --git a/libpod/adapter/runtime_remote.go b/libpod/adapter/runtime_remote.go index b1d4d4d25..ca2fad852 100644 --- a/libpod/adapter/runtime_remote.go +++ b/libpod/adapter/runtime_remote.go @@ -20,6 +20,7 @@ import ( "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" + "github.com/containers/libpod/utils" "github.com/containers/storage/pkg/archive" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -385,8 +386,11 @@ func (r *LocalRuntime) Export(name string, path string) error { if err != nil { return err } + return r.GetFileFromRemoteHost(tempPath, path, true) +} - outputFile, err := os.Create(path) +func (r *LocalRuntime) GetFileFromRemoteHost(remoteFilePath, outputPath string, delete bool) error { + outputFile, err := os.Create(outputPath) if err != nil { return err } @@ -395,7 +399,7 @@ func (r *LocalRuntime) Export(name string, path string) error { writer := bufio.NewWriter(outputFile) defer writer.Flush() - reply, err := iopodman.ReceiveFile().Send(r.Conn, varlink.Upgrade, tempPath, true) + reply, err := iopodman.ReceiveFile().Send(r.Conn, varlink.Upgrade, remoteFilePath, delete) if err != nil { return err } @@ -409,7 +413,6 @@ func (r *LocalRuntime) Export(name string, path string) error { if _, err := io.CopyN(writer, reader, length); err != nil { return errors.Wrap(err, "file transer failed") } - return nil } @@ -467,34 +470,23 @@ func (r *LocalRuntime) Build(ctx context.Context, c *cliconfig.BuildValues, opti Squash: options.Squash, } // tar the file - logrus.Debugf("creating tarball of context dir %s", options.ContextDirectory) - input, err := archive.Tar(options.ContextDirectory, archive.Uncompressed) - if err != nil { - return errors.Wrapf(err, "unable to create tarball of context dir %s", options.ContextDirectory) - } - - // Write the tarball to the fs - // TODO we might considering sending this without writing to the fs for the sake of performance - // under given conditions like memory availability. outputFile, err := ioutil.TempFile("", "varlink_tar_send") if err != nil { return err } defer outputFile.Close() - logrus.Debugf("writing context dir tarball to %s", outputFile.Name()) + defer os.Remove(outputFile.Name()) - _, err = io.Copy(outputFile, input) - if err != nil { + // Create the tarball of the context dir to a tempfile + if err := utils.TarToFilesystem(options.ContextDirectory, outputFile); err != nil { return err } - - logrus.Debugf("completed writing context dir tarball %s", outputFile.Name()) // Send the context dir tarball over varlink. tempFile, err := r.SendFileOverVarlink(outputFile.Name()) if err != nil { return err } - buildinfo.ContextDir = strings.Replace(tempFile, ":", "", -1) + buildinfo.ContextDir = tempFile reply, err := iopodman.BuildImage().Send(r.Conn, varlink.More, buildinfo) if err != nil { @@ -557,7 +549,7 @@ func (r *LocalRuntime) SendFileOverVarlink(source string) (string, error) { } - return tempFile, nil + return strings.Replace(tempFile, ":", "", -1), nil } // GetAllVolumes retrieves all the volumes @@ -702,3 +694,105 @@ func (r *LocalRuntime) PruneVolumes(ctx context.Context) ([]string, []error) { } return prunedNames, errs } + +// SaveImage is a wrapper function for saving an image to the local filesystem +func (r *LocalRuntime) SaveImage(ctx context.Context, c *cliconfig.SaveValues) error { + source := c.InputArgs[0] + additionalTags := c.InputArgs[1:] + + options := iopodman.ImageSaveOptions{ + Name: source, + Format: c.Format, + Output: c.Output, + MoreTags: additionalTags, + Quiet: c.Quiet, + Compress: c.Compress, + } + reply, err := iopodman.ImageSave().Send(r.Conn, varlink.More, options) + if err != nil { + return err + } + + var fetchfile string + for { + responses, flags, err := reply() + if err != nil { + return err + } + if len(responses.Id) > 0 { + fetchfile = responses.Id + } + for _, line := range responses.Logs { + fmt.Print(line) + } + if flags&varlink.Continues == 0 { + break + } + + } + if err != nil { + return err + } + + outputToDir := false + outfile := c.Output + var outputFile *os.File + // If the result is supposed to be a dir, then we need to put the tarfile + // from the host in a temporary file + if options.Format != "oci-archive" && options.Format != "docker-archive" { + outputToDir = true + outputFile, err = ioutil.TempFile("", "saveimage_tempfile") + if err != nil { + return err + } + outfile = outputFile.Name() + defer outputFile.Close() + defer os.Remove(outputFile.Name()) + } + // We now need to fetch the tarball result back to the more system + if err := r.GetFileFromRemoteHost(fetchfile, outfile, true); err != nil { + return err + } + + // If the result is a tarball, we're done + // If it is a dir, we need to untar the temporary file into the dir + if outputToDir { + if err := utils.UntarToFileSystem(c.Output, outputFile, &archive.TarOptions{}); err != nil { + return err + } + } + return nil +} + +// LoadImage loads a container image from a remote client's filesystem +func (r *LocalRuntime) LoadImage(ctx context.Context, name string, cli *cliconfig.LoadValues) (string, error) { + var names string + remoteTempFile, err := r.SendFileOverVarlink(cli.Input) + if err != nil { + return "", nil + } + more := varlink.More + if cli.Quiet { + more = 0 + } + reply, err := iopodman.LoadImage().Send(r.Conn, uint64(more), name, remoteTempFile, cli.Quiet, true) + if err != nil { + return "", err + } + + for { + responses, flags, err := reply() + if err != nil { + logrus.Error(err) + return "", err + } + for _, line := range responses.Logs { + fmt.Print(line) + } + names = responses.Id + if flags&varlink.Continues == 0 { + break + } + } + return names, nil +} diff --git a/libpod/adapter/shortcuts/shortcuts.go b/libpod/adapter/shortcuts/shortcuts.go new file mode 100644 index 000000000..0633399ae --- /dev/null +++ b/libpod/adapter/shortcuts/shortcuts.go @@ -0,0 +1,27 @@ +package shortcuts + +import "github.com/containers/libpod/libpod" + +// GetPodsByContext gets pods whether all, latest, or a slice of names/ids +func GetPodsByContext(all, latest bool, pods []string, runtime *libpod.Runtime) ([]*libpod.Pod, error) { + var outpods []*libpod.Pod + if all { + return runtime.GetAllPods() + } + if latest { + p, err := runtime.GetLatestPod() + if err != nil { + return nil, err + } + outpods = append(outpods, p) + return outpods, nil + } + for _, p := range pods { + pod, err := runtime.LookupPod(p) + if err != nil { + return nil, err + } + outpods = append(outpods, pod) + } + return outpods, nil +} diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index 5bc15dd7f..25ef5cd0e 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -783,6 +783,94 @@ func (s *BoltState) AllContainers() ([]*Container, error) { return ctrs, nil } +// RewriteContainerConfig rewrites a container's configuration. +// WARNING: This function is DANGEROUS. Do not use without reading the full +// comment on this function in state.go. +func (s *BoltState) RewriteContainerConfig(ctr *Container, newCfg *ContainerConfig) error { + if !s.valid { + return ErrDBClosed + } + + if !ctr.valid { + return ErrCtrRemoved + } + + newCfgJSON, err := json.Marshal(newCfg) + if err != nil { + return errors.Wrapf(err, "error marshalling new configuration JSON for container %s", ctr.ID()) + } + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + ctrBkt, err := getCtrBucket(tx) + if err != nil { + return err + } + + ctrDB := ctrBkt.Bucket([]byte(ctr.ID())) + if ctrDB == nil { + ctr.valid = false + return errors.Wrapf(ErrNoSuchCtr, "no container with ID %s found in DB", ctr.ID()) + } + + if err := ctrDB.Put(configKey, newCfgJSON); err != nil { + return errors.Wrapf(err, "error updating container %s config JSON", ctr.ID()) + } + + return nil + }) + return err +} + +// RewritePodConfig rewrites a pod's configuration. +// WARNING: This function is DANGEROUS. Do not use without reading the full +// comment on this function in state.go. +func (s *BoltState) RewritePodConfig(pod *Pod, newCfg *PodConfig) error { + if !s.valid { + return ErrDBClosed + } + + if !pod.valid { + return ErrPodRemoved + } + + newCfgJSON, err := json.Marshal(newCfg) + if err != nil { + return errors.Wrapf(err, "error marshalling new configuration JSON for container %s", pod.ID()) + } + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + podBkt, err := getPodBucket(tx) + if err != nil { + return err + } + + podDB := podBkt.Bucket([]byte(pod.ID())) + if podDB == nil { + pod.valid = false + return errors.Wrapf(ErrNoSuchPod, "no pod with ID %s found in DB", pod.ID()) + } + + if err := podDB.Put(configKey, newCfgJSON); err != nil { + return errors.Wrapf(err, "error updating pod %s config JSON", pod.ID()) + } + + return nil + }) + return err +} + // Pod retrieves a pod given its full ID func (s *BoltState) Pod(id string) (*Pod, error) { if id == "" { @@ -1281,10 +1369,6 @@ func (s *BoltState) RemoveVolume(volume *Volume) error { return ErrDBClosed } - if !volume.valid { - return ErrVolumeRemoved - } - volName := []byte(volume.Name()) db, err := s.getDBCon() diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go index ea150cfac..3d749849d 100644 --- a/libpod/boltdb_state_internal.go +++ b/libpod/boltdb_state_internal.go @@ -348,13 +348,6 @@ func (s *BoltState) getVolumeFromDB(name []byte, volume *Volume, volBkt *bolt.Bu return errors.Wrapf(err, "error unmarshalling volume %s config from DB", string(name)) } - // Get the lock - lock, err := s.runtime.lockManager.RetrieveLock(volume.config.LockID) - if err != nil { - return errors.Wrapf(err, "error retrieving lockfile for volume %s", string(name)) - } - volume.lock = lock - volume.runtime = s.runtime volume.valid = true diff --git a/libpod/image/image.go b/libpod/image/image.go index 028a795ea..b20419d7b 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -5,14 +5,18 @@ import ( "encoding/json" "fmt" "io" + "os" "strings" "syscall" "time" types2 "github.com/containernetworking/cni/pkg/types" cp "github.com/containers/image/copy" + "github.com/containers/image/directory" + dockerarchive "github.com/containers/image/docker/archive" "github.com/containers/image/docker/reference" "github.com/containers/image/manifest" + ociarchive "github.com/containers/image/oci/archive" is "github.com/containers/image/storage" "github.com/containers/image/tarball" "github.com/containers/image/transports" @@ -26,8 +30,9 @@ import ( "github.com/containers/storage" "github.com/containers/storage/pkg/reexec" digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ociv1 "github.com/opencontainers/image-spec/specs-go/v1" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -1084,3 +1089,65 @@ func (i *Image) Comment(ctx context.Context, manifestType string) (string, error } return ociv1Img.History[0].Comment, nil } + +// Save writes a container image to the filesystem +func (i *Image) Save(ctx context.Context, source, format, output string, moreTags []string, quiet, compress bool) error { + var ( + writer io.Writer + destRef types.ImageReference + manifestType string + err error + ) + + if quiet { + writer = os.Stderr + } + switch format { + case "oci-archive": + destImageName := imageNameForSaveDestination(i, source) + destRef, err = ociarchive.NewReference(output, destImageName) // destImageName may be "" + if err != nil { + return errors.Wrapf(err, "error getting OCI archive ImageReference for (%q, %q)", output, destImageName) + } + case "oci-dir": + destRef, err = directory.NewReference(output) + if err != nil { + return errors.Wrapf(err, "error getting directory ImageReference for %q", output) + } + manifestType = imgspecv1.MediaTypeImageManifest + case "docker-dir": + destRef, err = directory.NewReference(output) + if err != nil { + return errors.Wrapf(err, "error getting directory ImageReference for %q", output) + } + manifestType = manifest.DockerV2Schema2MediaType + case "docker-archive", "": + dst := output + destImageName := imageNameForSaveDestination(i, source) + if destImageName != "" { + dst = fmt.Sprintf("%s:%s", dst, destImageName) + } + destRef, err = dockerarchive.ParseReference(dst) // FIXME? Add dockerarchive.NewReference + if err != nil { + return errors.Wrapf(err, "error getting Docker archive ImageReference for %q", dst) + } + default: + return errors.Errorf("unknown format option %q", format) + } + // supports saving multiple tags to the same tar archive + var additionaltags []reference.NamedTagged + if len(moreTags) > 0 { + additionaltags, err = GetAdditionalTags(moreTags) + if err != nil { + return err + } + } + if err := i.PushImageToReference(ctx, destRef, manifestType, "", "", writer, compress, SigningOptions{}, &DockerRegistryOptions{}, additionaltags); err != nil { + if err2 := os.Remove(output); err2 != nil { + logrus.Errorf("error deleting %q: %v", output, err) + } + return errors.Wrapf(err, "unable to save %q", source) + } + + return nil +} diff --git a/libpod/image/utils.go b/libpod/image/utils.go index 3585428ad..544796a4b 100644 --- a/libpod/image/utils.go +++ b/libpod/image/utils.go @@ -1,6 +1,7 @@ package image import ( + "fmt" "io" "net/url" "regexp" @@ -148,3 +149,28 @@ func IsValidImageURI(imguri string) (bool, error) { } return true, nil } + +// imageNameForSaveDestination returns a Docker-like reference appropriate for saving img, +// which the user referred to as imgUserInput; or an empty string, if there is no appropriate +// reference. +func imageNameForSaveDestination(img *Image, imgUserInput string) string { + if strings.Contains(img.ID(), imgUserInput) { + return "" + } + + prepend := "" + localRegistryPrefix := fmt.Sprintf("%s/", DefaultLocalRegistry) + if !strings.HasPrefix(imgUserInput, localRegistryPrefix) { + // we need to check if localhost was added to the image name in NewFromLocal + for _, name := range img.Names() { + // If the user is saving an image in the localhost registry, getLocalImage need + // a name that matches the format localhost/<tag1>:<tag2> or localhost/<tag>:latest to correctly + // set up the manifest and save. + if strings.HasPrefix(name, localRegistryPrefix) && (strings.HasSuffix(name, imgUserInput) || strings.HasSuffix(name, fmt.Sprintf("%s:latest", imgUserInput))) { + prepend = localRegistryPrefix + break + } + } + } + return fmt.Sprintf("%s%s", prepend, imgUserInput) +} diff --git a/libpod/in_memory_state.go b/libpod/in_memory_state.go index 314799309..ab4fc8ba7 100644 --- a/libpod/in_memory_state.go +++ b/libpod/in_memory_state.go @@ -378,6 +378,58 @@ func (s *InMemoryState) ContainerInUse(ctr *Container) ([]string, error) { return arr, nil } +// AllContainers retrieves all containers from the state +func (s *InMemoryState) AllContainers() ([]*Container, error) { + ctrs := make([]*Container, 0, len(s.containers)) + for _, ctr := range s.containers { + if s.namespace == "" || ctr.config.Namespace == s.namespace { + ctrs = append(ctrs, ctr) + } + } + + return ctrs, nil +} + +// RewriteContainerConfig rewrites a container's configuration. +// This function is DANGEROUS, even with an in-memory state. +// Please read the full comment on it in state.go before using it. +func (s *InMemoryState) RewriteContainerConfig(ctr *Container, newCfg *ContainerConfig) error { + if !ctr.valid { + return ErrCtrRemoved + } + + // If the container does not exist, return error + stateCtr, ok := s.containers[ctr.ID()] + if !ok { + ctr.valid = false + return errors.Wrapf(ErrNoSuchCtr, "container with ID %s not found in state", ctr.ID()) + } + + stateCtr.config = newCfg + + return nil +} + +// RewritePodConfig rewrites a pod's configuration. +// This function is DANGEROUS, even with in-memory state. +// Please read the full comment on it in state.go before using it. +func (s *InMemoryState) RewritePodConfig(pod *Pod, newCfg *PodConfig) error { + if !pod.valid { + return ErrPodRemoved + } + + // If the pod does not exist, return error + statePod, ok := s.pods[pod.ID()] + if !ok { + pod.valid = false + return errors.Wrapf(ErrNoSuchPod, "pod with ID %s not found in state", pod.ID()) + } + + statePod.config = newCfg + + return nil +} + // Volume retrieves a volume from its full name func (s *InMemoryState) Volume(name string) (*Volume, error) { if name == "" { @@ -486,18 +538,6 @@ func (s *InMemoryState) AllVolumes() ([]*Volume, error) { return allVols, nil } -// AllContainers retrieves all containers from the state -func (s *InMemoryState) AllContainers() ([]*Container, error) { - ctrs := make([]*Container, 0, len(s.containers)) - for _, ctr := range s.containers { - if s.namespace == "" || ctr.config.Namespace == s.namespace { - ctrs = append(ctrs, ctr) - } - } - - return ctrs, nil -} - // Pod retrieves a pod from the state from its full ID func (s *InMemoryState) Pod(id string) (*Pod, error) { if id == "" { diff --git a/libpod/lock/in_memory_locks.go b/libpod/lock/in_memory_locks.go index db8f20e95..7c9605917 100644 --- a/libpod/lock/in_memory_locks.go +++ b/libpod/lock/in_memory_locks.go @@ -89,3 +89,14 @@ func (m *InMemoryManager) RetrieveLock(id uint32) (Locker, error) { return m.locks[id], nil } + +// FreeAllLocks frees all locks. +// This function is DANGEROUS. Please read the full comment in locks.go before +// trying to use it. +func (m *InMemoryManager) FreeAllLocks() error { + for _, lock := range m.locks { + lock.allocated = false + } + + return nil +} diff --git a/libpod/lock/lock.go b/libpod/lock/lock.go index 1f94171fe..d6841646b 100644 --- a/libpod/lock/lock.go +++ b/libpod/lock/lock.go @@ -24,6 +24,20 @@ type Manager interface { // The underlying lock MUST be the same as another other lock with the // same UUID. RetrieveLock(id uint32) (Locker, error) + // PLEASE READ FULL DESCRIPTION BEFORE USING. + // FreeAllLocks frees all allocated locks, in preparation for lock + // reallocation. + // As this deallocates all presently-held locks, this can be very + // dangerous - if there are other processes running that might be + // attempting to allocate new locks and free existing locks, we may + // encounter races leading to an inconsistent state. + // (This is in addition to the fact that FreeAllLocks instantly makes + // the state inconsistent simply by using it, and requires a full + // lock renumbering to restore consistency!). + // In short, this should only be used as part of unit tests, or lock + // renumbering, where reasonable guarantees about other processes can be + // made. + FreeAllLocks() error } // Locker is similar to sync.Locker, but provides a method for freeing the lock diff --git a/libpod/lock/shm/shm_lock.c b/libpod/lock/shm/shm_lock.c index 4af58d857..d11fce71a 100644 --- a/libpod/lock/shm/shm_lock.c +++ b/libpod/lock/shm/shm_lock.c @@ -203,6 +203,8 @@ shm_struct_t *setup_lock_shm(char *path, uint32_t num_locks, int *error_code) { // terminating NULL byte. // Returns a valid pointer on success or NULL on error. // If an error occurs, negative ERRNO values will be written to error_code. +// ERANGE is returned for a mismatch between num_locks and the number of locks +// available in the the SHM lock struct. shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code) { int shm_fd; shm_struct_t *shm; @@ -255,11 +257,11 @@ shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code) { // Need to check the SHM to see if it's actually our locks if (shm->magic != MAGIC) { - *error_code = -1 * errno; + *error_code = -1 * EBADF; goto CLEANUP; } if (shm->num_locks != (num_bitmaps * BITMAP_SIZE)) { - *error_code = -1 * errno; + *error_code = -1 * ERANGE; goto CLEANUP; } @@ -407,6 +409,36 @@ int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index) { return 0; } +// Deallocate all semaphores unconditionally. +// Returns negative ERRNO values. +int32_t deallocate_all_semaphores(shm_struct_t *shm) { + int ret_code; + uint i; + + if (shm == NULL) { + return -1 * EINVAL; + } + + // Lock the mutex controlling access to our shared memory + ret_code = take_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } + + // Iterate through all bitmaps and reset to unused + for (i = 0; i < shm->num_bitmaps; i++) { + shm->locks[i].bitmap = 0; + } + + // Unlock the allocation control mutex + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } + + return 0; +} + // Lock a given semaphore // Does not check if the semaphore is allocated - this ensures that, even for // removed containers, we can still successfully lock to check status (and diff --git a/libpod/lock/shm/shm_lock.go b/libpod/lock/shm/shm_lock.go index 87d28e5c1..e70ea8743 100644 --- a/libpod/lock/shm/shm_lock.go +++ b/libpod/lock/shm/shm_lock.go @@ -155,6 +155,22 @@ func (locks *SHMLocks) DeallocateSemaphore(sem uint32) error { return nil } +// DeallocateAllSemaphores frees all semaphores so they can be reallocated to +// other containers and pods. +func (locks *SHMLocks) DeallocateAllSemaphores() error { + if !locks.valid { + return errors.Wrapf(syscall.EINVAL, "locks have already been closed") + } + + retCode := C.deallocate_all_semaphores(locks.lockStruct) + if retCode < 0 { + // Negative errno return from C + return syscall.Errno(-1 * retCode) + } + + return nil +} + // LockSemaphore locks the given semaphore. // If the semaphore is already locked, LockSemaphore will block until the lock // can be acquired. diff --git a/libpod/lock/shm/shm_lock.h b/libpod/lock/shm/shm_lock.h index 8e7e23fb7..58e4297e2 100644 --- a/libpod/lock/shm/shm_lock.h +++ b/libpod/lock/shm/shm_lock.h @@ -40,6 +40,7 @@ shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code); int32_t close_lock_shm(shm_struct_t *shm); int64_t allocate_semaphore(shm_struct_t *shm); int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index); +int32_t deallocate_all_semaphores(shm_struct_t *shm); int32_t lock_semaphore(shm_struct_t *shm, uint32_t sem_index); int32_t unlock_semaphore(shm_struct_t *shm, uint32_t sem_index); diff --git a/libpod/lock/shm/shm_lock_test.go b/libpod/lock/shm/shm_lock_test.go index 594eb5d8e..830035881 100644 --- a/libpod/lock/shm/shm_lock_test.go +++ b/libpod/lock/shm/shm_lock_test.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "runtime" - "syscall" "testing" "time" @@ -53,12 +52,8 @@ func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { } defer func() { // Deallocate all locks - // Ignore ENOENT (lock is not allocated) - var i uint32 - for i = 0; i < numLocks; i++ { - if err := locks.DeallocateSemaphore(i); err != nil && err != syscall.ENOENT { - t.Fatalf("Error deallocating semaphore %d: %v", i, err) - } + if err := locks.DeallocateAllSemaphores(); err != nil { + t.Fatalf("Error deallocating semaphores: %v", err) } if err := locks.Close(); err != nil { @@ -212,6 +207,25 @@ func TestAllocateDeallocateCycle(t *testing.T) { }) } +// Test that DeallocateAllSemaphores deallocates all semaphores +func TestDeallocateAllSemaphoresDeallocatesAll(t *testing.T) { + runLockTest(t, func(t *testing.T, locks *SHMLocks) { + // Allocate a lock + locks1, err := locks.AllocateSemaphore() + assert.NoError(t, err) + + // Free all locks + err = locks.DeallocateAllSemaphores() + assert.NoError(t, err) + + // Allocate another lock + locks2, err := locks.AllocateSemaphore() + assert.NoError(t, err) + + assert.Equal(t, locks1, locks2) + }) +} + // Test that locks actually lock func TestLockSemaphoreActuallyLocks(t *testing.T) { runLockTest(t, func(t *testing.T, locks *SHMLocks) { diff --git a/libpod/lock/shm_lock_manager_linux.go b/libpod/lock/shm_lock_manager_linux.go index 94dfd7dd7..8678958ee 100644 --- a/libpod/lock/shm_lock_manager_linux.go +++ b/libpod/lock/shm_lock_manager_linux.go @@ -71,6 +71,13 @@ func (m *SHMLockManager) RetrieveLock(id uint32) (Locker, error) { return lock, nil } +// FreeAllLocks frees all locks in the manager. +// This function is DANGEROUS. Please read the full comment in locks.go before +// trying to use it. +func (m *SHMLockManager) FreeAllLocks() error { + return m.locks.DeallocateAllSemaphores() +} + // SHMLock is an individual shared memory lock. type SHMLock struct { lockID uint32 diff --git a/libpod/lock/shm_lock_manager_unsupported.go b/libpod/lock/shm_lock_manager_unsupported.go index cbdb2f7bc..1d6e3fcbd 100644 --- a/libpod/lock/shm_lock_manager_unsupported.go +++ b/libpod/lock/shm_lock_manager_unsupported.go @@ -27,3 +27,8 @@ func (m *SHMLockManager) AllocateLock() (Locker, error) { func (m *SHMLockManager) RetrieveLock(id string) (Locker, error) { return nil, fmt.Errorf("not supported") } + +// FreeAllLocks is not supported on this platform +func (m *SHMLockManager) FreeAllLocks() error { + return fmt.Errorf("not supported") +} diff --git a/libpod/options.go b/libpod/options.go index 06737776b..7c37fd65b 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -394,6 +394,22 @@ func WithDefaultInfraCommand(cmd string) RuntimeOption { } } +// WithRenumber instructs libpod to perform a lock renumbering while +// initializing. This will handle migrations from early versions of libpod with +// file locks to newer versions with SHM locking, as well as changes in the +// number of configured locks. +func WithRenumber() RuntimeOption { + return func(rt *Runtime) error { + if rt.valid { + return ErrRuntimeFinalized + } + + rt.doRenumber = true + + return nil + } +} + // Container Creation Options // WithShmDir sets the directory that should be mounted on /dev/shm. diff --git a/libpod/runtime.go b/libpod/runtime.go index 4f5d1e292..94dbf37dd 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "sync" + "syscall" "github.com/BurntSushi/toml" is "github.com/containers/image/storage" @@ -79,7 +80,8 @@ type RuntimeOption func(*Runtime) error // Runtime is the core libpod runtime type Runtime struct { - config *RuntimeConfig + config *RuntimeConfig + state State store storage.Store storageService *storageService @@ -88,12 +90,23 @@ type Runtime struct { netPlugin ocicni.CNIPlugin ociRuntimePath OCIRuntimePath conmonPath string - valid bool - lock sync.RWMutex imageRuntime *image.Runtime firewallBackend firewall.FirewallBackend lockManager lock.Manager configuredFrom *runtimeConfiguredFrom + + // doRenumber indicates that the runtime should perform a lock renumber + // during initialization. + // Once the runtime has been initialized and returned, this variable is + // unused. + doRenumber bool + + // valid indicates whether the runtime is ready to use. + // valid is set to true when a runtime is returned from GetRuntime(), + // and remains true until the runtime is shut down (rendering its + // storage unusable). When valid is false, the runtime cannot be used. + valid bool + lock sync.RWMutex } // OCIRuntimePath contains information about an OCI runtime. @@ -753,6 +766,7 @@ func makeRuntime(runtime *Runtime) (err error) { aliveLock.Unlock() } }() + _, err = os.Stat(runtimeAliveFile) if err != nil { // If the file doesn't exist, we need to refresh the state @@ -778,12 +792,35 @@ func makeRuntime(runtime *Runtime) (err error) { if err != nil { return err } + } else if errors.Cause(err) == syscall.ERANGE && runtime.doRenumber { + logrus.Debugf("Number of locks does not match - removing old locks") + + // ERANGE indicates a lock numbering mismatch. + // Since we're renumbering, this is not fatal. + // Remove the earlier set of locks and recreate. + if err := os.Remove(filepath.Join("/dev/shm", lockPath)); err != nil { + return errors.Wrapf(err, "error removing libpod locks file %s", lockPath) + } + + manager, err = lock.NewSHMLockManager(lockPath, runtime.config.NumLocks) + if err != nil { + return err + } } else { return err } } runtime.lockManager = manager + // If we're renumbering locks, do it now. + // It breaks out of normal runtime init, and will not return a valid + // runtime. + if runtime.doRenumber { + if err := runtime.renumberLocks(); err != nil { + return err + } + } + // If we need to refresh the state, do it now - things are guaranteed to // be set up by now. if doRefresh { diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 1055da75b..2ec8d0795 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -421,7 +421,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool, for _, v := range volumes { if volume, err := runtime.state.Volume(v); err == nil { - if err := runtime.removeVolume(ctx, volume, false, true); err != nil && err != ErrNoSuchVolume && err != ErrVolumeBeingUsed { + if err := runtime.removeVolume(ctx, volume, false); err != nil && err != ErrNoSuchVolume && err != ErrVolumeBeingUsed { logrus.Errorf("cleanup volume (%s): %v", v, err) } } diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 1e9689362..451c2ebe7 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -3,7 +3,6 @@ package libpod import ( "context" "fmt" - "github.com/opencontainers/image-spec/specs-go/v1" "io" "io/ioutil" "net/http" @@ -15,6 +14,11 @@ import ( "github.com/containers/libpod/pkg/util" "github.com/containers/storage" "github.com/pkg/errors" + + "github.com/containers/image/directory" + dockerarchive "github.com/containers/image/docker/archive" + ociarchive "github.com/containers/image/oci/archive" + "github.com/opencontainers/image-spec/specs-go/v1" ) // Runtime API @@ -211,3 +215,41 @@ func downloadFromURL(source string) (string, error) { return outFile.Name(), nil } + +// LoadImage loads a container image into local storage +func (r *Runtime) LoadImage(ctx context.Context, name, inputFile string, writer io.Writer, signaturePolicy string) (string, error) { + var newImages []*image.Image + src, err := dockerarchive.ParseReference(inputFile) // FIXME? We should add dockerarchive.NewReference() + if err == nil { + newImages, err = r.ImageRuntime().LoadFromArchiveReference(ctx, src, signaturePolicy, writer) + } + if err != nil { + // generate full src name with specified image:tag + src, err := ociarchive.NewReference(inputFile, name) // imageName may be "" + if err == nil { + newImages, err = r.ImageRuntime().LoadFromArchiveReference(ctx, src, signaturePolicy, writer) + } + if err != nil { + src, err := directory.NewReference(inputFile) + if err == nil { + newImages, err = r.ImageRuntime().LoadFromArchiveReference(ctx, src, signaturePolicy, writer) + } + if err != nil { + return "", errors.Wrapf(err, "error pulling %q", name) + } + } + } + return getImageNames(newImages), nil +} + +func getImageNames(images []*image.Image) string { + var names string + for i := range images { + if i == 0 { + names = images[i].InputName + } else { + names += ", " + images[i].InputName + } + } + return names +} diff --git a/libpod/runtime_renumber.go b/libpod/runtime_renumber.go new file mode 100644 index 000000000..125cf0825 --- /dev/null +++ b/libpod/runtime_renumber.go @@ -0,0 +1,57 @@ +package libpod + +import ( + "github.com/pkg/errors" +) + +// renumberLocks reassigns lock numbers for all containers and pods in the +// state. +// TODO: It would be desirable to make it impossible to call this until all +// other libpod sessions are dead. +// Possibly use a read-write file lock, with all non-renumber podmans owning the +// lock as read, renumber attempting to take a write lock? +// The alternative is some sort of session tracking, and I don't know how +// reliable that can be. +func (r *Runtime) renumberLocks() error { + // Start off by deallocating all locks + if err := r.lockManager.FreeAllLocks(); err != nil { + return err + } + + allCtrs, err := r.state.AllContainers() + if err != nil { + return err + } + for _, ctr := range allCtrs { + lock, err := r.lockManager.AllocateLock() + if err != nil { + return errors.Wrapf(err, "error allocating lock for container %s", ctr.ID()) + } + + ctr.config.LockID = lock.ID() + + // Write the new lock ID + if err := r.state.RewriteContainerConfig(ctr, ctr.config); err != nil { + return err + } + } + allPods, err := r.state.AllPods() + if err != nil { + return err + } + for _, pod := range allPods { + lock, err := r.lockManager.AllocateLock() + if err != nil { + return errors.Wrapf(err, "error allocating lock for pod %s", pod.ID()) + } + + pod.config.LockID = lock.ID() + + // Write the new lock ID + if err := r.state.RewritePodConfig(pod, pod.config); err != nil { + return err + } + } + + return nil +} diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go index beae50ac9..11f37ad4b 100644 --- a/libpod/runtime_volume.go +++ b/libpod/runtime_volume.go @@ -19,7 +19,7 @@ type VolumeCreateOption func(*Volume) error type VolumeFilter func(*Volume) bool // RemoveVolume removes a volumes -func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force, prune bool) error { +func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool) error { r.lock.Lock() defer r.lock.Unlock() @@ -35,10 +35,7 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force, prune bool } } - v.lock.Lock() - defer v.lock.Unlock() - - return r.removeVolume(ctx, v, force, prune) + return r.removeVolume(ctx, v, force) } // RemoveVolumes removes a slice of volumes or all with a force bool @@ -64,7 +61,7 @@ func (r *Runtime) RemoveVolumes(ctx context.Context, volumes []string, all, forc } for _, vol := range vols { - if err := r.RemoveVolume(ctx, vol, force, false); err != nil { + if err := r.RemoveVolume(ctx, vol, force); err != nil { return deletedVols, err } logrus.Debugf("removed volume %s", vol.Name()) @@ -168,8 +165,8 @@ func (r *Runtime) PruneVolumes(ctx context.Context) ([]string, []error) { } for _, vol := range vols { - if err := r.RemoveVolume(ctx, vol, false, true); err != nil { - if err != ErrVolumeBeingUsed { + if err := r.RemoveVolume(ctx, vol, false); err != nil { + if errors.Cause(err) != ErrVolumeBeingUsed && errors.Cause(err) != ErrVolumeRemoved { pruneErrors = append(pruneErrors, err) } continue diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go index 0727cfedf..838c0167a 100644 --- a/libpod/runtime_volume_linux.go +++ b/libpod/runtime_volume_linux.go @@ -67,13 +67,6 @@ func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) } volume.config.MountPoint = fullVolPath - lock, err := r.lockManager.AllocateLock() - if err != nil { - return nil, errors.Wrapf(err, "error allocating lock for new volume") - } - volume.lock = lock - volume.config.LockID = volume.lock.ID() - volume.valid = true // Add the volume to state @@ -85,9 +78,12 @@ func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) } // removeVolume removes the specified volume from state as well tears down its mountpoint and storage -func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force, prune bool) error { +func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool) error { if !v.valid { - return ErrNoSuchVolume + if ok, _ := r.state.HasVolume(v.Name()); !ok { + return nil + } + return ErrVolumeRemoved } deps, err := r.state.VolumeInUse(v) @@ -95,9 +91,6 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force, prune bool return err } if len(deps) != 0 { - if prune { - return ErrVolumeBeingUsed - } depsStr := strings.Join(deps, ", ") if !force { return errors.Wrapf(ErrVolumeBeingUsed, "volume %s is being used by the following container(s): %s", v.Name(), depsStr) @@ -112,18 +105,20 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force, prune bool } } - // Delete the mountpoint path of the volume, that is delete the volume from /var/lib/containers/storage/volumes - if err := v.teardownStorage(); err != nil { - return errors.Wrapf(err, "error cleaning up volume storage for %q", v.Name()) - } + // Set volume as invalid so it can no longer be used + v.valid = false // Remove the volume from the state if err := r.state.RemoveVolume(v); err != nil { return errors.Wrapf(err, "error removing volume %s", v.Name()) } - // Set volume as invalid so it can no longer be used - v.valid = false + // Delete the mountpoint path of the volume, that is delete the volume from /var/lib/containers/storage/volumes + if err := v.teardownStorage(); err != nil { + return errors.Wrapf(err, "error cleaning up volume storage for %q", v.Name()) + } + + logrus.Debugf("Removed volume %s", v.Name()) return nil } diff --git a/libpod/state.go b/libpod/state.go index 88d89f673..98282fc83 100644 --- a/libpod/state.go +++ b/libpod/state.go @@ -97,6 +97,30 @@ type State interface { // returned. AllContainers() ([]*Container, error) + // PLEASE READ FULL DESCRIPTION BEFORE USING. + // Rewrite a container's configuration. + // This function breaks libpod's normal prohibition on a read-only + // configuration, and as such should be used EXTREMELY SPARINGLY and + // only in very specific circumstances. + // Specifically, it is ONLY safe to use thing function to make changes + // that result in a functionally identical configuration (migrating to + // newer, but identical, configuration fields), or during libpod init + // WHILE HOLDING THE ALIVE LOCK (to prevent other libpod instances from + // being initialized). + // Most things in config can be changed by this, but container ID and + // name ABSOLUTELY CANNOT BE ALTERED. If you do so, there is a high + // potential for database corruption. + // There are a lot of capital letters and conditions here, but the short + // answer is this: use this only very sparingly, and only if you really + // know what you're doing. + RewriteContainerConfig(ctr *Container, newCfg *ContainerConfig) error + // PLEASE READ THE ABOVE DESCRIPTION BEFORE USING. + // This function is identical to RewriteContainerConfig, save for the + // fact that it is used with pods instead. + // It is subject to the same conditions as RewriteContainerConfig. + // Please do not use this unless you know what you're doing. + RewritePodConfig(pod *Pod, newCfg *PodConfig) error + // Accepts full ID of pod. // If the pod given is not in the set namespace, an error will be // returned. diff --git a/libpod/state_test.go b/libpod/state_test.go index 4bd00ab55..be68a2d69 100644 --- a/libpod/state_test.go +++ b/libpod/state_test.go @@ -1298,6 +1298,78 @@ func TestCannotUseBadIDAsGenericDependency(t *testing.T) { }) } +func TestRewriteContainerConfigDoesNotExist(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + err := state.RewriteContainerConfig(&Container{}, &ContainerConfig{}) + assert.Error(t, err) + }) +} + +func TestRewriteContainerConfigNotInState(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + testCtr, err := getTestCtr1(manager) + assert.NoError(t, err) + err = state.RewriteContainerConfig(testCtr, &ContainerConfig{}) + assert.Error(t, err) + }) +} + +func TestRewriteContainerConfigRewritesConfig(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + testCtr, err := getTestCtr1(manager) + assert.NoError(t, err) + + err = state.AddContainer(testCtr) + assert.NoError(t, err) + + testCtr.config.LogPath = "/another/path/" + + err = state.RewriteContainerConfig(testCtr, testCtr.config) + assert.NoError(t, err) + + testCtrFromState, err := state.Container(testCtr.ID()) + assert.NoError(t, err) + + testContainersEqual(t, testCtrFromState, testCtr, true) + }) +} + +func TestRewritePodConfigDoesNotExist(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + err := state.RewritePodConfig(&Pod{}, &PodConfig{}) + assert.Error(t, err) + }) +} + +func TestRewritePodConfigNotInState(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + testPod, err := getTestPod1(manager) + assert.NoError(t, err) + err = state.RewritePodConfig(testPod, &PodConfig{}) + assert.Error(t, err) + }) +} + +func TestRewritePodConfigRewritesConfig(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + testPod, err := getTestPod1(manager) + assert.NoError(t, err) + + err = state.AddPod(testPod) + assert.NoError(t, err) + + testPod.config.CgroupParent = "/another_cgroup_parent" + + err = state.RewritePodConfig(testPod, testPod.config) + assert.NoError(t, err) + + testPodFromState, err := state.Pod(testPod.ID()) + assert.NoError(t, err) + + testPodsEqual(t, testPodFromState, testPod, true) + }) +} + func TestGetPodDoesNotExist(t *testing.T) { runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { _, err := state.Pod("doesnotexist") diff --git a/libpod/volume.go b/libpod/volume.go index 026a3bf49..74878b6a4 100644 --- a/libpod/volume.go +++ b/libpod/volume.go @@ -1,7 +1,5 @@ package libpod -import "github.com/containers/libpod/libpod/lock" - // Volume is the type used to create named volumes // TODO: all volumes should be created using this and the Volume API type Volume struct { @@ -9,7 +7,6 @@ type Volume struct { valid bool runtime *Runtime - lock lock.Locker } // VolumeConfig holds the volume's config information @@ -17,8 +14,6 @@ type Volume struct { type VolumeConfig struct { // Name of the volume Name string `json:"name"` - // ID of this volume's lock - LockID uint32 `json:"lockID"` Labels map[string]string `json:"labels"` MountPoint string `json:"mountPoint"` diff --git a/libpod/volume_internal.go b/libpod/volume_internal.go index 0de8a2350..35f0ca19d 100644 --- a/libpod/volume_internal.go +++ b/libpod/volume_internal.go @@ -18,8 +18,5 @@ func newVolume(runtime *Runtime) (*Volume, error) { // teardownStorage deletes the volume from volumePath func (v *Volume) teardownStorage() error { - if !v.valid { - return ErrNoSuchVolume - } return os.RemoveAll(filepath.Join(v.runtime.config.VolumePath, v.Name())) } diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index 279a03d3f..acc5d6b2a 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -12,6 +12,7 @@ #include <fcntl.h> #include <sys/wait.h> #include <string.h> +#include <stdbool.h> static const char *_max_user_namespaces = "/proc/sys/user/max_user_namespaces"; static const char *_unprivileged_user_namespaces = "/proc/sys/kernel/unprivileged_userns_clone"; @@ -186,6 +187,18 @@ reexec_in_user_namespace (int ready) pid_t ppid = getpid (); char **argv; char uid[16]; + char *listen_fds = NULL; + char *listen_pid = NULL; + bool do_socket_activation = false; + + listen_pid = getenv("LISTEN_PID"); + listen_fds = getenv("LISTEN_FDS"); + + if (listen_pid != NULL && listen_fds != NULL) { + if (strtol(listen_pid, NULL, 10) == getpid()) { + do_socket_activation = true; + } + } sprintf (uid, "%d", geteuid ()); @@ -197,8 +210,22 @@ reexec_in_user_namespace (int ready) check_proc_sys_userns_file (_max_user_namespaces); check_proc_sys_userns_file (_unprivileged_user_namespaces); } - if (pid) + if (pid) { + if (do_socket_activation) { + long num_fds; + num_fds = strtol(listen_fds, NULL, 10); + if (num_fds != LONG_MIN && num_fds != LONG_MAX) { + long i; + for (i = 0; i < num_fds; i++) { + close(3+i); + } + } + unsetenv("LISTEN_PID"); + unsetenv("LISTEN_FDS"); + unsetenv("LISTEN_FDNAMES"); + } return pid; + } argv = get_cmd_line_args (ppid); if (argv == NULL) @@ -207,6 +234,12 @@ reexec_in_user_namespace (int ready) _exit (EXIT_FAILURE); } + if (do_socket_activation) { + char s[32]; + sprintf(s, "%d", getpid()); + setenv("LISTEN_PID", s, true); + } + setenv ("_LIBPOD_USERNS_CONFIGURED", "init", 1); setenv ("_LIBPOD_ROOTLESS_UID", uid, 1); diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index d12ab97ab..a27bdb9e5 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -736,3 +736,169 @@ func (i *LibpodAPI) ImagesPrune(call iopodman.VarlinkCall, all bool) error { } return call.ReplyImagesPrune(prunedImages) } + +// ImageSave .... +func (i *LibpodAPI) ImageSave(call iopodman.VarlinkCall, options iopodman.ImageSaveOptions) error { + newImage, err := i.Runtime.ImageRuntime().NewFromLocal(options.Name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + + // Determine if we are dealing with a tarball or dir + var output string + outputToDir := false + if options.Format == "oci-archive" || options.Format == "docker-archive" { + tempfile, err := ioutil.TempFile("", "varlink_send") + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + output = tempfile.Name() + tempfile.Close() + } else { + var err error + outputToDir = true + output, err = ioutil.TempDir("", "varlink_send") + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + } + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + if call.WantsMore() { + call.Continues = true + } + + saveOutput := bytes.NewBuffer([]byte{}) + c := make(chan error) + go func() { + err := newImage.Save(getContext(), options.Name, options.Format, output, options.MoreTags, options.Quiet, options.Compress) + c <- err + close(c) + }() + var log []string + done := false + for { + line, err := saveOutput.ReadString('\n') + if err == nil { + log = append(log, line) + continue + } else if err == io.EOF { + select { + case err := <-c: + if err != nil { + logrus.Errorf("reading of output during save failed for %s", newImage.ID()) + return call.ReplyErrorOccurred(err.Error()) + } + done = true + default: + if !call.WantsMore() { + time.Sleep(1 * time.Second) + break + } + br := iopodman.MoreResponse{ + Logs: log, + } + call.ReplyImageSave(br) + log = []string{} + } + } else { + return call.ReplyErrorOccurred(err.Error()) + } + if done { + break + } + } + call.Continues = false + + sendfile := output + // Image has been saved to `output` + if outputToDir { + // If the output is a directory, we need to tar up the directory to send it back + //Create a tempfile for the directory tarball + outputFile, err := ioutil.TempFile("", "varlink_save_dir") + if err != nil { + return err + } + defer outputFile.Close() + if err := utils.TarToFilesystem(output, outputFile); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + sendfile = outputFile.Name() + } + br := iopodman.MoreResponse{ + Logs: log, + Id: sendfile, + } + return call.ReplyPushImage(br) +} + +// LoadImage ... +func (i *LibpodAPI) LoadImage(call iopodman.VarlinkCall, name, inputFile string, deleteInputFile, quiet bool) error { + var ( + names string + writer io.Writer + err error + ) + if !quiet { + writer = os.Stderr + } + + if call.WantsMore() { + call.Continues = true + } + output := bytes.NewBuffer([]byte{}) + + c := make(chan error) + go func() { + names, err = i.Runtime.LoadImage(getContext(), name, inputFile, writer, "") + c <- err + close(c) + }() + + var log []string + done := false + for { + line, err := output.ReadString('\n') + if err == nil { + log = append(log, line) + continue + } else if err == io.EOF { + select { + case err := <-c: + if err != nil { + logrus.Error(err) + return call.ReplyErrorOccurred(err.Error()) + } + done = true + default: + if !call.WantsMore() { + time.Sleep(1 * time.Second) + break + } + br := iopodman.MoreResponse{ + Logs: log, + } + call.ReplyLoadImage(br) + log = []string{} + } + } else { + return call.ReplyErrorOccurred(err.Error()) + } + if done { + break + } + } + call.Continues = false + + br := iopodman.MoreResponse{ + Logs: log, + Id: names, + } + if deleteInputFile { + if err := os.Remove(inputFile); err != nil { + logrus.Errorf("unable to delete input file %s", inputFile) + } + } + return call.ReplyLoadImage(br) +} diff --git a/pkg/varlinkapi/pods.go b/pkg/varlinkapi/pods.go index 6e758786a..1dd864d03 100644 --- a/pkg/varlinkapi/pods.go +++ b/pkg/varlinkapi/pods.go @@ -2,6 +2,7 @@ package varlinkapi import ( "encoding/json" + "github.com/containers/libpod/libpod/adapter/shortcuts" "github.com/containers/libpod/pkg/rootless" "syscall" @@ -271,3 +272,17 @@ func (i *LibpodAPI) GetPodStats(call iopodman.VarlinkCall, name string) error { } return call.ReplyGetPodStats(pod.ID(), containersStats) } + +// GetPodsByContext returns a slice of pod ids based on all, latest, or a list +func (i *LibpodAPI) GetPodsByContext(call iopodman.VarlinkCall, all, latest bool, input []string) error { + var podids []string + + pods, err := shortcuts.GetPodsByContext(all, latest, input, i.Runtime) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + for _, p := range pods { + podids = append(podids, p.ID()) + } + return call.ReplyGetPodsByContext(podids) +} diff --git a/test/e2e/load_test.go b/test/e2e/load_test.go index 423f99ac8..c85810454 100644 --- a/test/e2e/load_test.go +++ b/test/e2e/load_test.go @@ -199,11 +199,9 @@ var _ = Describe("Podman load", func() { It("podman load localhost registry from scratch and :latest", func() { outfile := filepath.Join(podmanTest.TempDir, "load_test.tar.gz") - setup := podmanTest.Podman([]string{"pull", fedoraMinimal}) - setup.WaitWithDefaultTimeout() - Expect(setup.ExitCode()).To(Equal(0)) + podmanTest.RestoreArtifact("fedora-minimal:latest") - setup = podmanTest.Podman([]string{"tag", "fedora-minimal", "hello"}) + setup := podmanTest.Podman([]string{"tag", "fedora-minimal", "hello"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) diff --git a/test/e2e/save_test.go b/test/e2e/save_test.go index b354492b8..9f64e49a7 100644 --- a/test/e2e/save_test.go +++ b/test/e2e/save_test.go @@ -1,5 +1,3 @@ -// +build !remoteclient - package integration import ( diff --git a/utils/utils.go b/utils/utils.go index 4a91b304f..33b0eb1c5 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,12 +4,15 @@ import ( "bytes" "fmt" "io" + "os" "os/exec" "strings" + "github.com/containers/storage/pkg/archive" systemdDbus "github.com/coreos/go-systemd/dbus" "github.com/godbus/dbus" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ExecCmd executes a command with args and returns its output as a string along @@ -139,3 +142,30 @@ func CopyDetachable(dst io.Writer, src io.Reader, keys []byte) (written int64, e } return written, err } + +// UntarToFileSystem untars an os.file of a tarball to a destination in the filesystem +func UntarToFileSystem(dest string, tarball *os.File, options *archive.TarOptions) error { + logrus.Debugf("untarring %s", tarball.Name()) + return archive.Untar(tarball, dest, options) +} + +// TarToFilesystem creates a tarball from source and writes to an os.file +// provided +func TarToFilesystem(source string, tarball *os.File) error { + tb, err := Tar(source) + if err != nil { + return err + } + _, err = io.Copy(tarball, tb) + if err != nil { + return err + } + logrus.Debugf("wrote tarball file %s", tarball.Name()) + return nil +} + +// Tar creates a tarball from source and returns a readcloser of it +func Tar(source string) (io.ReadCloser, error) { + logrus.Debugf("creating tarball of %s", source) + return archive.Tar(source, archive.Uncompressed) +} diff --git a/vendor.conf b/vendor.conf index 309189c05..ff1a9c4d5 100644 --- a/vendor.conf +++ b/vendor.conf @@ -15,8 +15,11 @@ github.com/containerd/cgroups 39b18af02c4120960f517a3a4c2588fabb61d02c github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins v0.7.4 -github.com/containers/image 93bced01015eb94bec4821df1876314be8197680 -github.com/containers/storage 06b6c2e4cf254f5922a79da058c94ac2a65bb92f +github.com/containers/image v1.4 +github.com/vbauerster/mpb v3.3.4 +github.com/mattn/go-isatty v0.0.4 +github.com/VividCortex/ewma v1.1.1 +github.com/containers/storage v1.10 github.com/containers/psgo v1.1 github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c @@ -46,7 +49,6 @@ github.com/imdario/mergo v0.3.6 github.com/json-iterator/go 1.1.5 github.com/modern-go/concurrent 1.0.3 github.com/modern-go/reflect2 v1.0.1 -github.com/mattn/go-runewidth v0.0.4 github.com/mistifyio/go-zfs v2.1.1 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 @@ -83,7 +85,6 @@ golang.org/x/sys 7fbe1cd0fcc20051e1fcb87fbabec4a1bacaaeba https://github.com/gol golang.org/x/text e6919f6577db79269a6443b9dc46d18f2238fb5d https://github.com/golang/text golang.org/x/time 85acf8d2951cb2a3bde7632f9ff273ef0379bcbd https://github.com/golang/time golang.org/x/sync 37e7f081c4d4c64e13b10787722085407fe5d15f https://github.com/golang/sync -gopkg.in/cheggaaa/pb.v1 v1.0.27 gopkg.in/inf.v0 v0.9.1 gopkg.in/mgo.v2 v2 gopkg.in/yaml.v2 v2.2.2 diff --git a/vendor/github.com/mattn/go-runewidth/LICENSE b/vendor/github.com/VividCortex/ewma/LICENSE index 91b5cef30..a78d643ed 100644 --- a/vendor/github.com/mattn/go-runewidth/LICENSE +++ b/vendor/github.com/VividCortex/ewma/LICENSE @@ -1,6 +1,6 @@ -The MIT License (MIT) +The MIT License -Copyright (c) 2016 Yasuhiro Matsumoto +Copyright (c) 2013 VividCortex Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/VividCortex/ewma/README.md b/vendor/github.com/VividCortex/ewma/README.md new file mode 100644 index 000000000..7aab61b87 --- /dev/null +++ b/vendor/github.com/VividCortex/ewma/README.md @@ -0,0 +1,140 @@ +# EWMA [](https://godoc.org/github.com/VividCortex/ewma)  + +This repo provides Exponentially Weighted Moving Average algorithms, or EWMAs for short, [based on our +Quantifying Abnormal Behavior talk](https://vividcortex.com/blog/2013/07/23/a-fast-go-library-for-exponential-moving-averages/). + +### Exponentially Weighted Moving Average + +An exponentially weighted moving average is a way to continuously compute a type of +average for a series of numbers, as the numbers arrive. After a value in the series is +added to the average, its weight in the average decreases exponentially over time. This +biases the average towards more recent data. EWMAs are useful for several reasons, chiefly +their inexpensive computational and memory cost, as well as the fact that they represent +the recent central tendency of the series of values. + +The EWMA algorithm requires a decay factor, alpha. The larger the alpha, the more the average +is biased towards recent history. The alpha must be between 0 and 1, and is typically +a fairly small number, such as 0.04. We will discuss the choice of alpha later. + +The algorithm works thus, in pseudocode: + +1. Multiply the next number in the series by alpha. +2. Multiply the current value of the average by 1 minus alpha. +3. Add the result of steps 1 and 2, and store it as the new current value of the average. +4. Repeat for each number in the series. + +There are special-case behaviors for how to initialize the current value, and these vary +between implementations. One approach is to start with the first value in the series; +another is to average the first 10 or so values in the series using an arithmetic average, +and then begin the incremental updating of the average. Each method has pros and cons. + +It may help to look at it pictorially. Suppose the series has five numbers, and we choose +alpha to be 0.50 for simplicity. Here's the series, with numbers in the neighborhood of 300. + + + +Now let's take the moving average of those numbers. First we set the average to the value +of the first number. + + + +Next we multiply the next number by alpha, multiply the current value by 1-alpha, and add +them to generate a new value. + + + +This continues until we are done. + + + +Notice how each of the values in the series decays by half each time a new value +is added, and the top of the bars in the lower portion of the image represents the +size of the moving average. It is a smoothed, or low-pass, average of the original +series. + +For further reading, see [Exponentially weighted moving average](http://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average) on wikipedia. + +### Choosing Alpha + +Consider a fixed-size sliding-window moving average (not an exponentially weighted moving average) +that averages over the previous N samples. What is the average age of each sample? It is N/2. + +Now suppose that you wish to construct a EWMA whose samples have the same average age. The formula +to compute the alpha required for this is: alpha = 2/(N+1). Proof is in the book +"Production and Operations Analysis" by Steven Nahmias. + +So, for example, if you have a time-series with samples once per second, and you want to get the +moving average over the previous minute, you should use an alpha of .032786885. This, by the way, +is the constant alpha used for this repository's SimpleEWMA. + +### Implementations + +This repository contains two implementations of the EWMA algorithm, with different properties. + +The implementations all conform to the MovingAverage interface, and the constructor returns +that type. + +Current implementations assume an implicit time interval of 1.0 between every sample added. +That is, the passage of time is treated as though it's the same as the arrival of samples. +If you need time-based decay when samples are not arriving precisely at set intervals, then +this package will not support your needs at present. + +#### SimpleEWMA + +A SimpleEWMA is designed for low CPU and memory consumption. It **will** have different behavior than the VariableEWMA +for multiple reasons. It has no warm-up period and it uses a constant +decay. These properties let it use less memory. It will also behave +differently when it's equal to zero, which is assumed to mean +uninitialized, so if a value is likely to actually become zero over time, +then any non-zero value will cause a sharp jump instead of a small change. + +#### VariableEWMA + +Unlike SimpleEWMA, this supports a custom age which must be stored, and thus uses more memory. +It also has a "warmup" time when you start adding values to it. It will report a value of 0.0 +until you have added the required number of samples to it. It uses some memory to store the +number of samples added to it. As a result it uses a little over twice the memory of SimpleEWMA. + +## Usage + +### API Documentation + +View the GoDoc generated documentation [here](http://godoc.org/github.com/VividCortex/ewma). + +```go +package main +import "github.com/VividCortex/ewma" + +func main() { + samples := [100]float64{ + 4599, 5711, 4746, 4621, 5037, 4218, 4925, 4281, 5207, 5203, 5594, 5149, + } + + e := ewma.NewMovingAverage() //=> Returns a SimpleEWMA if called without params + a := ewma.NewMovingAverage(5) //=> returns a VariableEWMA with a decay of 2 / (5 + 1) + + for _, f := range samples { + e.Add(f) + a.Add(f) + } + + e.Value() //=> 13.577404704631077 + a.Value() //=> 1.5806140565521463e-12 +} +``` + +## Contributing + +We only accept pull requests for minor fixes or improvements. This includes: + +* Small bug fixes +* Typos +* Documentation or comments + +Please open issues to discuss new features. Pull requests for new features will be rejected, +so we recommend forking the repository and making changes in your fork for your use case. + +## License + +This repository is Copyright (c) 2013 VividCortex, Inc. All rights reserved. +It is licensed under the MIT license. Please see the LICENSE file for applicable license terms. diff --git a/vendor/github.com/VividCortex/ewma/ewma.go b/vendor/github.com/VividCortex/ewma/ewma.go new file mode 100644 index 000000000..44d5d53e3 --- /dev/null +++ b/vendor/github.com/VividCortex/ewma/ewma.go @@ -0,0 +1,126 @@ +// Package ewma implements exponentially weighted moving averages. +package ewma + +// Copyright (c) 2013 VividCortex, Inc. All rights reserved. +// Please see the LICENSE file for applicable license terms. + +const ( + // By default, we average over a one-minute period, which means the average + // age of the metrics in the period is 30 seconds. + AVG_METRIC_AGE float64 = 30.0 + + // The formula for computing the decay factor from the average age comes + // from "Production and Operations Analysis" by Steven Nahmias. + DECAY float64 = 2 / (float64(AVG_METRIC_AGE) + 1) + + // For best results, the moving average should not be initialized to the + // samples it sees immediately. The book "Production and Operations + // Analysis" by Steven Nahmias suggests initializing the moving average to + // the mean of the first 10 samples. Until the VariableEwma has seen this + // many samples, it is not "ready" to be queried for the value of the + // moving average. This adds some memory cost. + WARMUP_SAMPLES uint8 = 10 +) + +// MovingAverage is the interface that computes a moving average over a time- +// series stream of numbers. The average may be over a window or exponentially +// decaying. +type MovingAverage interface { + Add(float64) + Value() float64 + Set(float64) +} + +// NewMovingAverage constructs a MovingAverage that computes an average with the +// desired characteristics in the moving window or exponential decay. If no +// age is given, it constructs a default exponentially weighted implementation +// that consumes minimal memory. The age is related to the decay factor alpha +// by the formula given for the DECAY constant. It signifies the average age +// of the samples as time goes to infinity. +func NewMovingAverage(age ...float64) MovingAverage { + if len(age) == 0 || age[0] == AVG_METRIC_AGE { + return new(SimpleEWMA) + } + return &VariableEWMA{ + decay: 2 / (age[0] + 1), + } +} + +// A SimpleEWMA represents the exponentially weighted moving average of a +// series of numbers. It WILL have different behavior than the VariableEWMA +// for multiple reasons. It has no warm-up period and it uses a constant +// decay. These properties let it use less memory. It will also behave +// differently when it's equal to zero, which is assumed to mean +// uninitialized, so if a value is likely to actually become zero over time, +// then any non-zero value will cause a sharp jump instead of a small change. +// However, note that this takes a long time, and the value may just +// decays to a stable value that's close to zero, but which won't be mistaken +// for uninitialized. See http://play.golang.org/p/litxBDr_RC for example. +type SimpleEWMA struct { + // The current value of the average. After adding with Add(), this is + // updated to reflect the average of all values seen thus far. + value float64 +} + +// Add adds a value to the series and updates the moving average. +func (e *SimpleEWMA) Add(value float64) { + if e.value == 0 { // this is a proxy for "uninitialized" + e.value = value + } else { + e.value = (value * DECAY) + (e.value * (1 - DECAY)) + } +} + +// Value returns the current value of the moving average. +func (e *SimpleEWMA) Value() float64 { + return e.value +} + +// Set sets the EWMA's value. +func (e *SimpleEWMA) Set(value float64) { + e.value = value +} + +// VariableEWMA represents the exponentially weighted moving average of a series of +// numbers. Unlike SimpleEWMA, it supports a custom age, and thus uses more memory. +type VariableEWMA struct { + // The multiplier factor by which the previous samples decay. + decay float64 + // The current value of the average. + value float64 + // The number of samples added to this instance. + count uint8 +} + +// Add adds a value to the series and updates the moving average. +func (e *VariableEWMA) Add(value float64) { + switch { + case e.count < WARMUP_SAMPLES: + e.count++ + e.value += value + case e.count == WARMUP_SAMPLES: + e.count++ + e.value = e.value / float64(WARMUP_SAMPLES) + e.value = (value * e.decay) + (e.value * (1 - e.decay)) + default: + e.value = (value * e.decay) + (e.value * (1 - e.decay)) + } +} + +// Value returns the current value of the average, or 0.0 if the series hasn't +// warmed up yet. +func (e *VariableEWMA) Value() float64 { + if e.count <= WARMUP_SAMPLES { + return 0.0 + } + + return e.value +} + +// Set sets the EWMA's value. +func (e *VariableEWMA) Set(value float64) { + e.value = value + if e.count <= WARMUP_SAMPLES { + e.count = WARMUP_SAMPLES + 1 + } +} diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 2d3a2a1a8..ba99336aa 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -22,12 +22,13 @@ import ( "github.com/containers/image/transports" "github.com/containers/image/types" "github.com/klauspost/pgzip" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/vbauerster/mpb" + "github.com/vbauerster/mpb/decor" "golang.org/x/crypto/ssh/terminal" "golang.org/x/sync/semaphore" - pb "gopkg.in/cheggaaa/pb.v1" ) type digestingReader struct { @@ -422,27 +423,6 @@ func (ic *imageCopier) updateEmbeddedDockerReference() error { return nil } -// shortDigest returns the first 12 characters of the digest. -func shortDigest(d digest.Digest) string { - return d.Encoded()[:12] -} - -// createProgressBar creates a pb.ProgressBar. Note that if the copier's -// reportWriter is ioutil.Discard, the progress bar's output will be discarded -// and a single line will be printed instead. -func (c *copier) createProgressBar(srcInfo types.BlobInfo, kind string) *pb.ProgressBar { - bar := pb.New(int(srcInfo.Size)).SetUnits(pb.U_BYTES) - bar.SetMaxWidth(80) - bar.ShowTimeLeft = false - bar.ShowPercent = false - bar.Prefix(fmt.Sprintf("Copying %s %s:", kind, shortDigest(srcInfo.Digest))) - bar.Output = c.progressOutput - if bar.Output == ioutil.Discard { - c.Printf("Copying %s %s\n", kind, srcInfo.Digest) - } - return bar -} - // isTTY returns true if the io.Writer is a file and a tty. func isTTY(w io.Writer) bool { if f, ok := w.(*os.File); ok { @@ -477,6 +457,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // copyGroup is used to determine if all layers are copied copyGroup := sync.WaitGroup{} copyGroup.Add(numLayers) + // copySemaphore is used to limit the number of parallel downloads to // avoid malicious images causing troubles and to be nice to servers. var copySemaphore *semaphore.Weighted @@ -487,8 +468,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { } data := make([]copyLayerData, numLayers) - copyLayerHelper := func(index int, srcLayer types.BlobInfo, bar *pb.ProgressBar) { - defer bar.Finish() + copyLayerHelper := func(index int, srcLayer types.BlobInfo, bar *mpb.Bar) { defer copySemaphore.Release(1) defer copyGroup.Done() cld := copyLayerData{} @@ -498,45 +478,37 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // does not support them. if ic.diffIDsAreNeeded { cld.err = errors.New("getting DiffID for foreign layers is unimplemented") - bar.Prefix(fmt.Sprintf("Skipping blob %s (DiffID foreign layer unimplemented):", shortDigest(srcLayer.Digest))) - bar.Finish() } else { cld.destInfo = srcLayer logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name()) - bar.Prefix(fmt.Sprintf("Skipping blob %s (foreign layer):", shortDigest(srcLayer.Digest))) - bar.Add64(bar.Total) - bar.Finish() } } else { cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, bar) } data[index] = cld + bar.SetTotal(srcLayer.Size, true) } - progressBars := make([]*pb.ProgressBar, numLayers) - for i, srcInfo := range srcInfos { - bar := ic.c.createProgressBar(srcInfo, "blob") - progressBars[i] = bar - } + func() { // A scope for defer + progressPool, progressCleanup := ic.c.newProgressPool(ctx) + defer progressCleanup() - progressPool := pb.NewPool(progressBars...) - progressPool.Output = ic.c.progressOutput + progressBars := make([]*mpb.Bar, numLayers) + for i, srcInfo := range srcInfos { + progressBars[i] = ic.c.createProgressBar(progressPool, srcInfo, "blob") + } - if err := progressPool.Start(); err != nil { - return errors.Wrapf(err, "error creating progress-bar pool") - } + for i, srcLayer := range srcInfos { + copySemaphore.Acquire(ctx, 1) + go copyLayerHelper(i, srcLayer, progressBars[i]) + } - for i, srcLayer := range srcInfos { - copySemaphore.Acquire(ctx, 1) - go copyLayerHelper(i, srcLayer, progressBars[i]) - } + // Wait for all layers to be copied + copyGroup.Wait() + }() destInfos := make([]types.BlobInfo, numLayers) diffIDs := make([]digest.Digest, numLayers) - - copyGroup.Wait() - progressPool.Stop() - for i, cld := range data { if cld.err != nil { return cld.err @@ -607,6 +579,44 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte return manifest, nil } +// newProgressPool creates a *mpb.Progress and a cleanup function. +// The caller must eventually call the returned cleanup function after the pool will no longer be updated. +func (c *copier) newProgressPool(ctx context.Context) (*mpb.Progress, func()) { + ctx, cancel := context.WithCancel(ctx) + pool := mpb.New(mpb.WithWidth(40), mpb.WithOutput(c.progressOutput), mpb.WithContext(ctx)) + return pool, func() { + cancel() + pool.Wait() + } +} + +// createProgressBar creates a mpb.Bar in pool. Note that if the copier's reportWriter +// is ioutil.Discard, the progress bar's output will be discarded +func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind string) *mpb.Bar { + // shortDigestLen is the length of the digest used for blobs. + const shortDigestLen = 12 + + prefix := fmt.Sprintf("Copying %s %s", kind, info.Digest.Encoded()) + // Truncate the prefix (chopping of some part of the digest) to make all progress bars aligned in a column. + maxPrefixLen := len("Copying blob ") + shortDigestLen + if len(prefix) > maxPrefixLen { + prefix = prefix[:maxPrefixLen] + } + + bar := pool.AddBar(info.Size, + mpb.PrependDecorators( + decor.Name(prefix), + ), + mpb.AppendDecorators( + decor.CountersKibiByte("%.1f / %.1f"), + ), + ) + if c.progressOutput == ioutil.Discard { + c.Printf("Copying %s %s\n", kind, info.Digest) + } + return bar +} + // copyConfig copies config.json, if any, from src to dest. func (c *copier) copyConfig(ctx context.Context, src types.Image) error { srcInfo := src.ConfigInfo() @@ -615,12 +625,20 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error { if err != nil { return errors.Wrapf(err, "Error reading config blob %s", srcInfo.Digest) } - bar := c.createProgressBar(srcInfo, "config") - defer bar.Finish() - bar.Start() - destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, bar) + + destInfo, err := func() (types.BlobInfo, error) { // A scope for defer + progressPool, progressCleanup := c.newProgressPool(ctx) + defer progressCleanup() + bar := c.createProgressBar(progressPool, srcInfo, "config") + destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, bar) + if err != nil { + return types.BlobInfo{}, err + } + bar.SetTotal(int64(len(configBlob)), true) + return destInfo, nil + }() if err != nil { - return err + return nil } if destInfo.Digest != srcInfo.Digest { return errors.Errorf("Internal error: copying uncompressed config blob %s changed digest to %s", srcInfo.Digest, destInfo.Digest) @@ -638,7 +656,7 @@ type diffIDResult struct { // copyLayer copies a layer with srcInfo (with known Digest and possibly known Size) in src to dest, perhaps compressing it if canCompress, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded -func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, bar *pb.ProgressBar) (types.BlobInfo, digest.Digest, error) { +func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, bar *mpb.Bar) (types.BlobInfo, digest.Digest, error) { cachedDiffID := ic.c.blobInfoCache.UncompressedDigest(srcInfo.Digest) // May be "" diffIDIsNeeded := ic.diffIDsAreNeeded && cachedDiffID == "" @@ -649,9 +667,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, ba return types.BlobInfo{}, "", errors.Wrapf(err, "Error trying to reuse blob %s at destination", srcInfo.Digest) } if reused { - bar.Prefix(fmt.Sprintf("Skipping blob %s (already present):", shortDigest(srcInfo.Digest))) - bar.Add64(bar.Total) - bar.Finish() + logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest) return blobInfo, cachedDiffID, nil } } @@ -663,8 +679,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, ba } defer srcStream.Close() - blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize}, - diffIDIsNeeded, bar) + blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize}, diffIDIsNeeded, bar) if err != nil { return types.BlobInfo{}, "", err } @@ -692,7 +707,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, ba // perhaps compressing the stream if canCompress, // and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller. func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, - diffIDIsNeeded bool, bar *pb.ProgressBar) (types.BlobInfo, <-chan diffIDResult, error) { + diffIDIsNeeded bool, bar *mpb.Bar) (types.BlobInfo, <-chan diffIDResult, error) { var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil var diffIDChan chan diffIDResult @@ -753,7 +768,7 @@ func computeDiffID(stream io.Reader, decompressor compression.DecompressorFunc) // and returns a complete blobInfo of the copied blob. func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer, - canModifyBlob bool, isConfig bool, bar *pb.ProgressBar) (types.BlobInfo, error) { + canModifyBlob bool, isConfig bool, bar *mpb.Bar) (types.BlobInfo, error) { // The copying happens through a pipeline of connected io.Readers. // === Input: srcStream @@ -776,7 +791,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest) } isCompressed := decompressor != nil - destStream = bar.NewProxyReader(destStream) + destStream = bar.ProxyReader(destStream) // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index bbbb1a458..1c5b6b378 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -28,7 +28,6 @@ golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8 golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08 -gopkg.in/cheggaaa/pb.v1 v1.0.27 gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a k8s.io/client-go bcde30fb7eaed76fd98a36b4120321b94995ffb6 github.com/xeipuuv/gojsonschema master @@ -48,3 +47,6 @@ github.com/boltdb/bolt master github.com/klauspost/pgzip v1.2.1 github.com/klauspost/compress v1.4.1 github.com/klauspost/cpuid v1.2.0 +github.com/vbauerster/mpb v3.3.4 +github.com/mattn/go-isatty v0.0.4 +github.com/VividCortex/ewma v1.1.1 diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index beaf41f39..10d628dbe 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -568,6 +568,9 @@ func (r *containerStore) Lock() { r.lockfile.Lock() } +func (r *containerStore) RLock() { + r.lockfile.RLock() +} func (r *containerStore) Unlock() { r.lockfile.Unlock() } diff --git a/vendor/github.com/containers/storage/containers_ffjson.go b/vendor/github.com/containers/storage/containers_ffjson.go index 40b912bb3..aef6becfe 100644 --- a/vendor/github.com/containers/storage/containers_ffjson.go +++ b/vendor/github.com/containers/storage/containers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./containers.go +// source: containers.go package storage diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 57d6dd63a..657d9b3ce 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -239,6 +239,8 @@ func parseOptions(options []string) (*overlayOptions, error) { } key = strings.ToLower(key) switch key { + case ".override_kernel_check", "overlay.override_kernel_check", "overlay2.override_kernel_check": + logrus.Warnf("overlay: override_kernel_check option was specified, but is no longer necessary") case ".mountopt", "overlay.mountopt", "overlay2.mountopt": o.mountOptions = val case ".size", "overlay.size", "overlay2.size": diff --git a/vendor/github.com/containers/storage/errors.go b/vendor/github.com/containers/storage/errors.go index 562415c09..63cb9ab74 100644 --- a/vendor/github.com/containers/storage/errors.go +++ b/vendor/github.com/containers/storage/errors.go @@ -43,8 +43,6 @@ var ( ErrSizeUnknown = errors.New("size is not known") // ErrStoreIsReadOnly is returned when the caller makes a call to a read-only store that would require modifying its contents. ErrStoreIsReadOnly = errors.New("called a write method on a read-only store") - // ErrLockReadOnly indicates that the caller only took a read-only lock, and is not allowed to write. - ErrLockReadOnly = errors.New("lock is not a read-write lock") // ErrDuplicateImageNames indicates that the read-only store uses the same name for multiple images. ErrDuplicateImageNames = errors.New("read-only image store assigns the same name to multiple images") // ErrDuplicateLayerNames indicates that the read-only store uses the same name for multiple layers. diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index fa4a7c43b..93505f5fb 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -756,6 +756,10 @@ func (r *imageStore) Lock() { r.lockfile.Lock() } +func (r *imageStore) RLock() { + r.lockfile.RLock() +} + func (r *imageStore) Unlock() { r.lockfile.Unlock() } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index cdc3cbba9..d612f0459 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -1222,6 +1222,10 @@ func (r *layerStore) Lock() { r.lockfile.Lock() } +func (r *layerStore) RLock() { + r.lockfile.RLock() +} + func (r *layerStore) Unlock() { r.lockfile.Unlock() } diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 09b5d0f33..125b5d8c9 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./layers.go +// source: layers.go package storage diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index 9f6a18144..7f07b9ac5 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -1,7 +1,6 @@ package storage import ( - "fmt" "path/filepath" "sync" "time" @@ -13,7 +12,14 @@ import ( // identifier of the last party that made changes to whatever's being protected // by the lock. type Locker interface { - sync.Locker + // Acquire a writer lock. + Lock() + + // Unlock the lock. + Unlock() + + // Acquire a reader lock. + RLock() // Touch records, for others sharing the lock, that the caller was the // last writer. It should only be called with the lock held. @@ -39,31 +45,22 @@ var ( ) // GetLockfile opens a read-write lock file, creating it if necessary. The -// Locker object it returns will be returned unlocked. +// Locker object may already be locked if the path has already been requested +// by the current process. func GetLockfile(path string) (Locker, error) { - lockfilesLock.Lock() - defer lockfilesLock.Unlock() - if lockfiles == nil { - lockfiles = make(map[string]Locker) - } - cleanPath := filepath.Clean(path) - if locker, ok := lockfiles[cleanPath]; ok { - if !locker.IsReadWrite() { - return nil, errors.Wrapf(ErrLockReadOnly, "lock %q is a read-only lock", cleanPath) - } - return locker, nil - } - locker, err := getLockFile(path, false) // platform dependent locker - if err != nil { - return nil, err - } - lockfiles[filepath.Clean(path)] = locker - return locker, nil + return getLockfile(path, false) } -// GetROLockfile opens a read-only lock file. The Locker object it returns -// will be returned unlocked. +// GetROLockfile opens a read-only lock file, creating it if necessary. The +// Locker object may already be locked if the path has already been requested +// by the current process. func GetROLockfile(path string) (Locker, error) { + return getLockfile(path, true) +} + +// getLockfile is a helper for GetLockfile and GetROLockfile and returns Locker +// based on the path and read-only property. +func getLockfile(path string, ro bool) (Locker, error) { lockfilesLock.Lock() defer lockfilesLock.Unlock() if lockfiles == nil { @@ -71,12 +68,15 @@ func GetROLockfile(path string) (Locker, error) { } cleanPath := filepath.Clean(path) if locker, ok := lockfiles[cleanPath]; ok { - if locker.IsReadWrite() { - return nil, fmt.Errorf("lock %q is a read-write lock", cleanPath) + if ro && locker.IsReadWrite() { + return nil, errors.Errorf("lock %q is not a read-only lock", cleanPath) + } + if !ro && !locker.IsReadWrite() { + return nil, errors.Errorf("lock %q is not a read-write lock", cleanPath) } return locker, nil } - locker, err := getLockFile(path, true) // platform dependent locker + locker, err := getLockFile(path, ro) // platform dependent locker if err != nil { return nil, err } diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 31e39c02e..0adbc49a5 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -3,6 +3,7 @@ package storage import ( + "fmt" "os" "sync" "time" @@ -24,38 +25,84 @@ func getLockFile(path string, ro bool) (Locker, error) { return nil, errors.Wrapf(err, "error opening %q", path) } unix.CloseOnExec(fd) + + locktype := unix.F_WRLCK if ro { - return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK, locked: false}, nil + locktype = unix.F_RDLCK } - return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK, locked: false}, nil + return &lockfile{ + stateMutex: &sync.Mutex{}, + writeMutex: &sync.Mutex{}, + file: path, + fd: uintptr(fd), + lw: stringid.GenerateRandomID(), + locktype: int16(locktype), + locked: false, + ro: ro}, nil } type lockfile struct { - mu sync.Mutex - file string - fd uintptr - lw string - locktype int16 - locked bool + // stateMutex is used to synchronize concurrent accesses + stateMutex *sync.Mutex + // writeMutex is used to serialize and avoid recursive writer locks + writeMutex *sync.Mutex + counter int64 + file string + fd uintptr + lw string + locktype int16 + locked bool + ro bool } -// Lock locks the lock file -func (l *lockfile) Lock() { +// lock locks the lockfile via FCTNL(2) based on the specified type and +// command. +func (l *lockfile) lock(l_type int16) { lk := unix.Flock_t{ - Type: l.locktype, + Type: l_type, Whence: int16(os.SEEK_SET), Start: 0, Len: 0, Pid: int32(os.Getpid()), } - l.mu.Lock() + if l_type == unix.F_WRLCK { + // If we try to lock as a writer, lock the writerMutex first to + // avoid multiple writer acquisitions of the same process. + // Note: it's important to lock it prior to the stateMutex to + // avoid a deadlock. + l.writeMutex.Lock() + } + l.stateMutex.Lock() + l.locktype = l_type + if l.counter == 0 { + // Optimization: only use the (expensive) fcntl syscall when + // the counter is 0. If it's greater than that, we're owning + // the lock already and can only be a reader. + for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { + time.Sleep(10 * time.Millisecond) + } + } l.locked = true - for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { - time.Sleep(10 * time.Millisecond) + l.counter++ + l.stateMutex.Unlock() +} + +// Lock locks the lockfile as a writer. Note that RLock() will be called if +// the lock is a read-only one. +func (l *lockfile) Lock() { + if l.ro { + l.RLock() + } else { + l.lock(unix.F_WRLCK) } } -// Unlock unlocks the lock file +// LockRead locks the lockfile as a reader. +func (l *lockfile) RLock() { + l.lock(unix.F_RDLCK) +} + +// Unlock unlocks the lockfile. func (l *lockfile) Unlock() { lk := unix.Flock_t{ Type: unix.F_UNLCK, @@ -64,19 +111,40 @@ func (l *lockfile) Unlock() { Len: 0, Pid: int32(os.Getpid()), } - for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { - time.Sleep(10 * time.Millisecond) + l.stateMutex.Lock() + if l.locked == false { + // Panic when unlocking an unlocked lock. That's a violation + // of the lock semantics and will reveal such. + panic("calling Unlock on unlocked lock") + } + l.counter-- + if l.counter < 0 { + // Panic when the counter is negative. There is no way we can + // recover from a corrupted lock and we need to protect the + // storage from corruption. + panic(fmt.Sprintf("lock %q has been unlocked too often", l.file)) + } + if l.counter == 0 { + // We should only release the lock when the counter is 0 to + // avoid releasing read-locks too early; a given process may + // acquire a read lock multiple times. + l.locked = false + for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { + time.Sleep(10 * time.Millisecond) + } + } + if l.locktype == unix.F_WRLCK { + l.writeMutex.Unlock() } - l.locked = false - l.mu.Unlock() + l.stateMutex.Unlock() } -// Check if lock is locked +// Locked checks if lockfile is locked. func (l *lockfile) Locked() bool { return l.locked } -// Touch updates the lock file with the UID of the user +// Touch updates the lock file with the UID of the user. func (l *lockfile) Touch() error { l.lw = stringid.GenerateRandomID() id := []byte(l.lw) @@ -98,7 +166,8 @@ func (l *lockfile) Touch() error { return nil } -// Modified indicates if the lock file has been updated since the last time it was loaded +// Modified indicates if the lockfile has been updated since the last time it +// was loaded. func (l *lockfile) Modified() (bool, error) { id := []byte(l.lw) _, err := unix.Seek(int(l.fd), 0, os.SEEK_SET) @@ -117,7 +186,7 @@ func (l *lockfile) Modified() (bool, error) { return l.lw != lw, nil } -// IsRWLock indicates if the lock file is a read-write lock +// IsReadWriteLock indicates if the lock file is a read-write lock. func (l *lockfile) IsReadWrite() bool { - return (l.locktype == unix.F_WRLCK) + return !l.ro } diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index 77490b851..a3821bfeb 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -23,6 +23,11 @@ func (l *lockfile) Lock() { l.locked = true } +func (l *lockfile) RLock() { + l.mu.Lock() + l.locked = true +} + func (l *lockfile) Unlock() { l.locked = false l.mu.Unlock() diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go b/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go index 2ba44b85d..9b8103e4d 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./pkg/archive/archive.go +// source: pkg/archive/archive.go package archive diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 856c73e51..d53703d6b 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -566,10 +566,10 @@ func GetStore(options StoreOptions) (Store, error) { } if options.GraphRoot == "" { - return nil, ErrIncompleteOptions + return nil, errors.Wrap(ErrIncompleteOptions, "no storage root specified") } if options.RunRoot == "" { - return nil, ErrIncompleteOptions + return nil, errors.Wrap(ErrIncompleteOptions, "no storage runroot specified") } if err := os.MkdirAll(options.RunRoot, 0700); err != nil && !os.IsExist(err) { @@ -870,7 +870,8 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w gidMap := options.GIDMap if parent != "" { var ilayer *Layer - for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) { + for _, l := range append([]ROLayerStore{rlstore}, rlstores...) { + lstore := l if lstore != rlstore { lstore.Lock() defer lstore.Unlock() @@ -949,7 +950,8 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o return nil, err } var ilayer *Layer - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1010,7 +1012,8 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, crea } var layer, parentLayer *Layer // Locate the image's top layer and its parent, if it has one. - for _, store := range append([]ROLayerStore{rlstore}, lstores...) { + for _, s := range append([]ROLayerStore{rlstore}, lstores...) { + store := s if store != rlstore { store.Lock() defer store.Unlock() @@ -1143,7 +1146,8 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } } var cimage *Image - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1315,7 +1319,8 @@ func (s *store) Metadata(id string) (string, error) { if err != nil { return "", err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1336,7 +1341,8 @@ func (s *store) Metadata(id string) (string, error) { if err != nil { return "", err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1375,7 +1381,8 @@ func (s *store) ListImageBigData(id string) ([]string, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1400,7 +1407,8 @@ func (s *store) ImageBigDataSize(id, key string) (int64, error) { if err != nil { return -1, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1426,7 +1434,8 @@ func (s *store) ImageBigDataDigest(id, key string) (digest.Digest, error) { return "", err } stores = append([]ROImageStore{ristore}, stores...) - for _, ristore := range stores { + for _, r := range stores { + ristore := r ristore.Lock() defer ristore.Unlock() if modified, err := ristore.Modified(); modified || err != nil { @@ -1451,8 +1460,9 @@ func (s *store) ImageBigData(id, key string) ([]byte, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { - store.Lock() + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s + store.RLock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { if err = store.Load(); err != nil { @@ -1495,7 +1505,8 @@ func (s *store) ImageSize(id string) (int64, error) { if err != nil { return -1, errors.Wrapf(err, "error loading additional layer stores") } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1516,7 +1527,8 @@ func (s *store) ImageSize(id string) (int64, error) { } // Look for the image's record. - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1603,7 +1615,8 @@ func (s *store) ContainerSize(id string) (int64, error) { if err != nil { return -1, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1772,7 +1785,8 @@ func (s *store) Exists(id string) bool { if err != nil { return false } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1793,7 +1807,8 @@ func (s *store) Exists(id string) bool { if err != nil { return false } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1895,7 +1910,8 @@ func (s *store) Names(id string) ([]string, error) { if err != nil { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1916,7 +1932,8 @@ func (s *store) Names(id string) ([]string, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1955,7 +1972,8 @@ func (s *store) Lookup(name string) (string, error) { if err != nil { return "", err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -1976,7 +1994,8 @@ func (s *store) Lookup(name string) (string, error) { if err != nil { return "", err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2486,7 +2505,8 @@ func (s *store) Changes(from, to string) ([]archive.Change, error) { if err != nil { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2510,7 +2530,8 @@ func (s *store) DiffSize(from, to string) (int64, error) { if err != nil { return -1, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2534,8 +2555,9 @@ func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, erro if err != nil { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { - store.Lock() + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s + store.RLock() if modified, err := store.Modified(); modified || err != nil { if err = store.Load(); err != nil { return nil, err @@ -2588,7 +2610,8 @@ func (s *store) layersByMappedDigest(m func(ROLayerStore, digest.Digest) ([]Laye if err != nil { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2634,7 +2657,8 @@ func (s *store) LayerSize(id string) (int64, error) { if err != nil { return -1, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2712,7 +2736,8 @@ func (s *store) Layers() ([]Layer, error) { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2740,7 +2765,8 @@ func (s *store) Images() ([]Image, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2783,7 +2809,8 @@ func (s *store) Layer(id string) (*Layer, error) { if err != nil { return nil, err } - for _, store := range append([]ROLayerStore{lstore}, lstores...) { + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2808,7 +2835,8 @@ func (s *store) Image(id string) (*Image, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { @@ -2840,7 +2868,8 @@ func (s *store) ImagesByTopLayer(id string) ([]*Image, error) { if err != nil { return nil, err } - for _, store := range append([]ROImageStore{istore}, istores...) { + for _, s := range append([]ROImageStore{istore}, istores...) { + store := s store.Lock() defer store.Unlock() if modified, err := store.Modified(); modified || err != nil { diff --git a/vendor/github.com/mattn/go-isatty/LICENSE b/vendor/github.com/mattn/go-isatty/LICENSE new file mode 100644 index 000000000..65dc692b6 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) Yasuhiro MATSUMOTO <mattn.jp@gmail.com> + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/mattn/go-isatty/README.md b/vendor/github.com/mattn/go-isatty/README.md new file mode 100644 index 000000000..1e69004bb --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/README.md @@ -0,0 +1,50 @@ +# go-isatty + +[](http://godoc.org/github.com/mattn/go-isatty) +[](https://travis-ci.org/mattn/go-isatty) +[](https://coveralls.io/github/mattn/go-isatty?branch=master) +[](https://goreportcard.com/report/mattn/go-isatty) + +isatty for golang + +## Usage + +```go +package main + +import ( + "fmt" + "github.com/mattn/go-isatty" + "os" +) + +func main() { + if isatty.IsTerminal(os.Stdout.Fd()) { + fmt.Println("Is Terminal") + } else if isatty.IsCygwinTerminal(os.Stdout.Fd()) { + fmt.Println("Is Cygwin/MSYS2 Terminal") + } else { + fmt.Println("Is Not Terminal") + } +} +``` + +## Installation + +``` +$ go get github.com/mattn/go-isatty +``` + +## License + +MIT + +## Author + +Yasuhiro Matsumoto (a.k.a mattn) + +## Thanks + +* k-takata: base idea for IsCygwinTerminal + + https://github.com/k-takata/go-iscygpty diff --git a/vendor/github.com/mattn/go-isatty/doc.go b/vendor/github.com/mattn/go-isatty/doc.go new file mode 100644 index 000000000..17d4f90eb --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/doc.go @@ -0,0 +1,2 @@ +// Package isatty implements interface to isatty +package isatty diff --git a/vendor/github.com/mattn/go-isatty/isatty_appengine.go b/vendor/github.com/mattn/go-isatty/isatty_appengine.go new file mode 100644 index 000000000..9584a9884 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_appengine.go @@ -0,0 +1,15 @@ +// +build appengine + +package isatty + +// IsTerminal returns true if the file descriptor is terminal which +// is always false on on appengine classic which is a sandboxed PaaS. +func IsTerminal(fd uintptr) bool { + return false +} + +// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_bsd.go b/vendor/github.com/mattn/go-isatty/isatty_bsd.go new file mode 100644 index 000000000..42f2514d1 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_bsd.go @@ -0,0 +1,18 @@ +// +build darwin freebsd openbsd netbsd dragonfly +// +build !appengine + +package isatty + +import ( + "syscall" + "unsafe" +) + +const ioctlReadTermios = syscall.TIOCGETA + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_linux.go b/vendor/github.com/mattn/go-isatty/isatty_linux.go new file mode 100644 index 000000000..7384cf991 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_linux.go @@ -0,0 +1,18 @@ +// +build linux +// +build !appengine,!ppc64,!ppc64le + +package isatty + +import ( + "syscall" + "unsafe" +) + +const ioctlReadTermios = syscall.TCGETS + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_linux_ppc64x.go b/vendor/github.com/mattn/go-isatty/isatty_linux_ppc64x.go new file mode 100644 index 000000000..44e5d2130 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_linux_ppc64x.go @@ -0,0 +1,19 @@ +// +build linux +// +build ppc64 ppc64le + +package isatty + +import ( + "unsafe" + + syscall "golang.org/x/sys/unix" +) + +const ioctlReadTermios = syscall.TCGETS + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + var termios syscall.Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_others.go b/vendor/github.com/mattn/go-isatty/isatty_others.go new file mode 100644 index 000000000..9d8b4a599 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_others.go @@ -0,0 +1,10 @@ +// +build !windows +// +build !appengine + +package isatty + +// IsCygwinTerminal return true if the file descriptor is a cygwin or msys2 +// terminal. This is also always false on this environment. +func IsCygwinTerminal(fd uintptr) bool { + return false +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_solaris.go b/vendor/github.com/mattn/go-isatty/isatty_solaris.go new file mode 100644 index 000000000..1f0c6bf53 --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_solaris.go @@ -0,0 +1,16 @@ +// +build solaris +// +build !appengine + +package isatty + +import ( + "golang.org/x/sys/unix" +) + +// IsTerminal returns true if the given file descriptor is a terminal. +// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c +func IsTerminal(fd uintptr) bool { + var termio unix.Termio + err := unix.IoctlSetTermio(int(fd), unix.TCGETA, &termio) + return err == nil +} diff --git a/vendor/github.com/mattn/go-isatty/isatty_windows.go b/vendor/github.com/mattn/go-isatty/isatty_windows.go new file mode 100644 index 000000000..af51cbcaa --- /dev/null +++ b/vendor/github.com/mattn/go-isatty/isatty_windows.go @@ -0,0 +1,94 @@ +// +build windows +// +build !appengine + +package isatty + +import ( + "strings" + "syscall" + "unicode/utf16" + "unsafe" +) + +const ( + fileNameInfo uintptr = 2 + fileTypePipe = 3 +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + procGetConsoleMode = kernel32.NewProc("GetConsoleMode") + procGetFileInformationByHandleEx = kernel32.NewProc("GetFileInformationByHandleEx") + procGetFileType = kernel32.NewProc("GetFileType") +) + +func init() { + // Check if GetFileInformationByHandleEx is available. + if procGetFileInformationByHandleEx.Find() != nil { + procGetFileInformationByHandleEx = nil + } +} + +// IsTerminal return true if the file descriptor is terminal. +func IsTerminal(fd uintptr) bool { + var st uint32 + r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, fd, uintptr(unsafe.Pointer(&st)), 0) + return r != 0 && e == 0 +} + +// Check pipe name is used for cygwin/msys2 pty. +// Cygwin/MSYS2 PTY has a name like: +// \{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master +func isCygwinPipeName(name string) bool { + token := strings.Split(name, "-") + if len(token) < 5 { + return false + } + + if token[0] != `\msys` && token[0] != `\cygwin` { + return false + } + + if token[1] == "" { + return false + } + + if !strings.HasPrefix(token[2], "pty") { + return false + } + + if token[3] != `from` && token[3] != `to` { + return false + } + + if token[4] != "master" { + return false + } + + return true +} + +// IsCygwinTerminal() return true if the file descriptor is a cygwin or msys2 +// terminal. +func IsCygwinTerminal(fd uintptr) bool { + if procGetFileInformationByHandleEx == nil { + return false + } + + // Cygwin/msys's pty is a pipe. + ft, _, e := syscall.Syscall(procGetFileType.Addr(), 1, fd, 0, 0) + if ft != fileTypePipe || e != 0 { + return false + } + + var buf [2 + syscall.MAX_PATH]uint16 + r, _, e := syscall.Syscall6(procGetFileInformationByHandleEx.Addr(), + 4, fd, fileNameInfo, uintptr(unsafe.Pointer(&buf)), + uintptr(len(buf)*2), 0, 0) + if r == 0 || e != 0 { + return false + } + + l := *(*uint32)(unsafe.Pointer(&buf)) + return isCygwinPipeName(string(utf16.Decode(buf[2 : 2+l/2]))) +} diff --git a/vendor/github.com/mattn/go-runewidth/README.mkd b/vendor/github.com/mattn/go-runewidth/README.mkd deleted file mode 100644 index 66663a94b..000000000 --- a/vendor/github.com/mattn/go-runewidth/README.mkd +++ /dev/null @@ -1,27 +0,0 @@ -go-runewidth -============ - -[](https://travis-ci.org/mattn/go-runewidth) -[](https://coveralls.io/r/mattn/go-runewidth?branch=HEAD) -[](http://godoc.org/github.com/mattn/go-runewidth) -[](https://goreportcard.com/report/github.com/mattn/go-runewidth) - -Provides functions to get fixed width of the character or string. - -Usage ------ - -```go -runewidth.StringWidth("つのだ☆HIRO") == 12 -``` - - -Author ------- - -Yasuhiro Matsumoto - -License -------- - -under the MIT License: http://mattn.mit-license.org/2013 diff --git a/vendor/github.com/mattn/go-runewidth/runewidth.go b/vendor/github.com/mattn/go-runewidth/runewidth.go deleted file mode 100644 index 3cb94106f..000000000 --- a/vendor/github.com/mattn/go-runewidth/runewidth.go +++ /dev/null @@ -1,977 +0,0 @@ -package runewidth - -import ( - "os" -) - -var ( - // EastAsianWidth will be set true if the current locale is CJK - EastAsianWidth bool - - // ZeroWidthJoiner is flag to set to use UTR#51 ZWJ - ZeroWidthJoiner bool - - // DefaultCondition is a condition in current locale - DefaultCondition = &Condition{} -) - -func init() { - handleEnv() -} - -func handleEnv() { - env := os.Getenv("RUNEWIDTH_EASTASIAN") - if env == "" { - EastAsianWidth = IsEastAsian() - } else { - EastAsianWidth = env == "1" - } - // update DefaultCondition - DefaultCondition.EastAsianWidth = EastAsianWidth - DefaultCondition.ZeroWidthJoiner = ZeroWidthJoiner -} - -type interval struct { - first rune - last rune -} - -type table []interval - -func inTables(r rune, ts ...table) bool { - for _, t := range ts { - if inTable(r, t) { - return true - } - } - return false -} - -func inTable(r rune, t table) bool { - // func (t table) IncludesRune(r rune) bool { - if r < t[0].first { - return false - } - - bot := 0 - top := len(t) - 1 - for top >= bot { - mid := (bot + top) >> 1 - - switch { - case t[mid].last < r: - bot = mid + 1 - case t[mid].first > r: - top = mid - 1 - default: - return true - } - } - - return false -} - -var private = table{ - {0x00E000, 0x00F8FF}, {0x0F0000, 0x0FFFFD}, {0x100000, 0x10FFFD}, -} - -var nonprint = table{ - {0x0000, 0x001F}, {0x007F, 0x009F}, {0x00AD, 0x00AD}, - {0x070F, 0x070F}, {0x180B, 0x180E}, {0x200B, 0x200F}, - {0x2028, 0x202E}, {0x206A, 0x206F}, {0xD800, 0xDFFF}, - {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0xFFFE, 0xFFFF}, -} - -var combining = table{ - {0x0300, 0x036F}, {0x0483, 0x0489}, {0x0591, 0x05BD}, - {0x05BF, 0x05BF}, {0x05C1, 0x05C2}, {0x05C4, 0x05C5}, - {0x05C7, 0x05C7}, {0x0610, 0x061A}, {0x064B, 0x065F}, - {0x0670, 0x0670}, {0x06D6, 0x06DC}, {0x06DF, 0x06E4}, - {0x06E7, 0x06E8}, {0x06EA, 0x06ED}, {0x0711, 0x0711}, - {0x0730, 0x074A}, {0x07A6, 0x07B0}, {0x07EB, 0x07F3}, - {0x0816, 0x0819}, {0x081B, 0x0823}, {0x0825, 0x0827}, - {0x0829, 0x082D}, {0x0859, 0x085B}, {0x08D4, 0x08E1}, - {0x08E3, 0x0903}, {0x093A, 0x093C}, {0x093E, 0x094F}, - {0x0951, 0x0957}, {0x0962, 0x0963}, {0x0981, 0x0983}, - {0x09BC, 0x09BC}, {0x09BE, 0x09C4}, {0x09C7, 0x09C8}, - {0x09CB, 0x09CD}, {0x09D7, 0x09D7}, {0x09E2, 0x09E3}, - {0x0A01, 0x0A03}, {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, - {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, - {0x0A70, 0x0A71}, {0x0A75, 0x0A75}, {0x0A81, 0x0A83}, - {0x0ABC, 0x0ABC}, {0x0ABE, 0x0AC5}, {0x0AC7, 0x0AC9}, - {0x0ACB, 0x0ACD}, {0x0AE2, 0x0AE3}, {0x0B01, 0x0B03}, - {0x0B3C, 0x0B3C}, {0x0B3E, 0x0B44}, {0x0B47, 0x0B48}, - {0x0B4B, 0x0B4D}, {0x0B56, 0x0B57}, {0x0B62, 0x0B63}, - {0x0B82, 0x0B82}, {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, - {0x0BCA, 0x0BCD}, {0x0BD7, 0x0BD7}, {0x0C00, 0x0C03}, - {0x0C3E, 0x0C44}, {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, - {0x0C55, 0x0C56}, {0x0C62, 0x0C63}, {0x0C81, 0x0C83}, - {0x0CBC, 0x0CBC}, {0x0CBE, 0x0CC4}, {0x0CC6, 0x0CC8}, - {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, {0x0CE2, 0x0CE3}, - {0x0D01, 0x0D03}, {0x0D3E, 0x0D44}, {0x0D46, 0x0D48}, - {0x0D4A, 0x0D4D}, {0x0D57, 0x0D57}, {0x0D62, 0x0D63}, - {0x0D82, 0x0D83}, {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, - {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, {0x0DF2, 0x0DF3}, - {0x0E31, 0x0E31}, {0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, - {0x0EB1, 0x0EB1}, {0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, - {0x0EC8, 0x0ECD}, {0x0F18, 0x0F19}, {0x0F35, 0x0F35}, - {0x0F37, 0x0F37}, {0x0F39, 0x0F39}, {0x0F3E, 0x0F3F}, - {0x0F71, 0x0F84}, {0x0F86, 0x0F87}, {0x0F8D, 0x0F97}, - {0x0F99, 0x0FBC}, {0x0FC6, 0x0FC6}, {0x102B, 0x103E}, - {0x1056, 0x1059}, {0x105E, 0x1060}, {0x1062, 0x1064}, - {0x1067, 0x106D}, {0x1071, 0x1074}, {0x1082, 0x108D}, - {0x108F, 0x108F}, {0x109A, 0x109D}, {0x135D, 0x135F}, - {0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753}, - {0x1772, 0x1773}, {0x17B4, 0x17D3}, {0x17DD, 0x17DD}, - {0x180B, 0x180D}, {0x1885, 0x1886}, {0x18A9, 0x18A9}, - {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1A17, 0x1A1B}, - {0x1A55, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A7F}, - {0x1AB0, 0x1ABE}, {0x1B00, 0x1B04}, {0x1B34, 0x1B44}, - {0x1B6B, 0x1B73}, {0x1B80, 0x1B82}, {0x1BA1, 0x1BAD}, - {0x1BE6, 0x1BF3}, {0x1C24, 0x1C37}, {0x1CD0, 0x1CD2}, - {0x1CD4, 0x1CE8}, {0x1CED, 0x1CED}, {0x1CF2, 0x1CF4}, - {0x1CF8, 0x1CF9}, {0x1DC0, 0x1DF5}, {0x1DFB, 0x1DFF}, - {0x20D0, 0x20F0}, {0x2CEF, 0x2CF1}, {0x2D7F, 0x2D7F}, - {0x2DE0, 0x2DFF}, {0x302A, 0x302F}, {0x3099, 0x309A}, - {0xA66F, 0xA672}, {0xA674, 0xA67D}, {0xA69E, 0xA69F}, - {0xA6F0, 0xA6F1}, {0xA802, 0xA802}, {0xA806, 0xA806}, - {0xA80B, 0xA80B}, {0xA823, 0xA827}, {0xA880, 0xA881}, - {0xA8B4, 0xA8C5}, {0xA8E0, 0xA8F1}, {0xA926, 0xA92D}, - {0xA947, 0xA953}, {0xA980, 0xA983}, {0xA9B3, 0xA9C0}, - {0xA9E5, 0xA9E5}, {0xAA29, 0xAA36}, {0xAA43, 0xAA43}, - {0xAA4C, 0xAA4D}, {0xAA7B, 0xAA7D}, {0xAAB0, 0xAAB0}, - {0xAAB2, 0xAAB4}, {0xAAB7, 0xAAB8}, {0xAABE, 0xAABF}, - {0xAAC1, 0xAAC1}, {0xAAEB, 0xAAEF}, {0xAAF5, 0xAAF6}, - {0xABE3, 0xABEA}, {0xABEC, 0xABED}, {0xFB1E, 0xFB1E}, - {0xFE00, 0xFE0F}, {0xFE20, 0xFE2F}, {0x101FD, 0x101FD}, - {0x102E0, 0x102E0}, {0x10376, 0x1037A}, {0x10A01, 0x10A03}, - {0x10A05, 0x10A06}, {0x10A0C, 0x10A0F}, {0x10A38, 0x10A3A}, - {0x10A3F, 0x10A3F}, {0x10AE5, 0x10AE6}, {0x11000, 0x11002}, - {0x11038, 0x11046}, {0x1107F, 0x11082}, {0x110B0, 0x110BA}, - {0x11100, 0x11102}, {0x11127, 0x11134}, {0x11173, 0x11173}, - {0x11180, 0x11182}, {0x111B3, 0x111C0}, {0x111CA, 0x111CC}, - {0x1122C, 0x11237}, {0x1123E, 0x1123E}, {0x112DF, 0x112EA}, - {0x11300, 0x11303}, {0x1133C, 0x1133C}, {0x1133E, 0x11344}, - {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11357, 0x11357}, - {0x11362, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, - {0x11435, 0x11446}, {0x114B0, 0x114C3}, {0x115AF, 0x115B5}, - {0x115B8, 0x115C0}, {0x115DC, 0x115DD}, {0x11630, 0x11640}, - {0x116AB, 0x116B7}, {0x1171D, 0x1172B}, {0x11C2F, 0x11C36}, - {0x11C38, 0x11C3F}, {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, - {0x16AF0, 0x16AF4}, {0x16B30, 0x16B36}, {0x16F51, 0x16F7E}, - {0x16F8F, 0x16F92}, {0x1BC9D, 0x1BC9E}, {0x1D165, 0x1D169}, - {0x1D16D, 0x1D172}, {0x1D17B, 0x1D182}, {0x1D185, 0x1D18B}, - {0x1D1AA, 0x1D1AD}, {0x1D242, 0x1D244}, {0x1DA00, 0x1DA36}, - {0x1DA3B, 0x1DA6C}, {0x1DA75, 0x1DA75}, {0x1DA84, 0x1DA84}, - {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, - {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, - {0x1E026, 0x1E02A}, {0x1E8D0, 0x1E8D6}, {0x1E944, 0x1E94A}, - {0xE0100, 0xE01EF}, -} - -var doublewidth = table{ - {0x1100, 0x115F}, {0x231A, 0x231B}, {0x2329, 0x232A}, - {0x23E9, 0x23EC}, {0x23F0, 0x23F0}, {0x23F3, 0x23F3}, - {0x25FD, 0x25FE}, {0x2614, 0x2615}, {0x2648, 0x2653}, - {0x267F, 0x267F}, {0x2693, 0x2693}, {0x26A1, 0x26A1}, - {0x26AA, 0x26AB}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, - {0x26CE, 0x26CE}, {0x26D4, 0x26D4}, {0x26EA, 0x26EA}, - {0x26F2, 0x26F3}, {0x26F5, 0x26F5}, {0x26FA, 0x26FA}, - {0x26FD, 0x26FD}, {0x2705, 0x2705}, {0x270A, 0x270B}, - {0x2728, 0x2728}, {0x274C, 0x274C}, {0x274E, 0x274E}, - {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, - {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, {0x2B1B, 0x2B1C}, - {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x2E80, 0x2E99}, - {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, - {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, - {0x3105, 0x312D}, {0x3131, 0x318E}, {0x3190, 0x31BA}, - {0x31C0, 0x31E3}, {0x31F0, 0x321E}, {0x3220, 0x3247}, - {0x3250, 0x32FE}, {0x3300, 0x4DBF}, {0x4E00, 0xA48C}, - {0xA490, 0xA4C6}, {0xA960, 0xA97C}, {0xAC00, 0xD7A3}, - {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, - {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, - {0xFFE0, 0xFFE6}, {0x16FE0, 0x16FE0}, {0x17000, 0x187EC}, - {0x18800, 0x18AF2}, {0x1B000, 0x1B001}, {0x1F004, 0x1F004}, - {0x1F0CF, 0x1F0CF}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, - {0x1F200, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, - {0x1F250, 0x1F251}, {0x1F300, 0x1F320}, {0x1F32D, 0x1F335}, - {0x1F337, 0x1F37C}, {0x1F37E, 0x1F393}, {0x1F3A0, 0x1F3CA}, - {0x1F3CF, 0x1F3D3}, {0x1F3E0, 0x1F3F0}, {0x1F3F4, 0x1F3F4}, - {0x1F3F8, 0x1F43E}, {0x1F440, 0x1F440}, {0x1F442, 0x1F4FC}, - {0x1F4FF, 0x1F53D}, {0x1F54B, 0x1F54E}, {0x1F550, 0x1F567}, - {0x1F57A, 0x1F57A}, {0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A4}, - {0x1F5FB, 0x1F64F}, {0x1F680, 0x1F6C5}, {0x1F6CC, 0x1F6CC}, - {0x1F6D0, 0x1F6D2}, {0x1F6EB, 0x1F6EC}, {0x1F6F4, 0x1F6F6}, - {0x1F910, 0x1F91E}, {0x1F920, 0x1F927}, {0x1F930, 0x1F930}, - {0x1F933, 0x1F93E}, {0x1F940, 0x1F94B}, {0x1F950, 0x1F95E}, - {0x1F980, 0x1F991}, {0x1F9C0, 0x1F9C0}, {0x20000, 0x2FFFD}, - {0x30000, 0x3FFFD}, -} - -var ambiguous = table{ - {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, - {0x00AA, 0x00AA}, {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, - {0x00B6, 0x00BA}, {0x00BC, 0x00BF}, {0x00C6, 0x00C6}, - {0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1}, - {0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED}, - {0x00F0, 0x00F0}, {0x00F2, 0x00F3}, {0x00F7, 0x00FA}, - {0x00FC, 0x00FC}, {0x00FE, 0x00FE}, {0x0101, 0x0101}, - {0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B}, - {0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133}, - {0x0138, 0x0138}, {0x013F, 0x0142}, {0x0144, 0x0144}, - {0x0148, 0x014B}, {0x014D, 0x014D}, {0x0152, 0x0153}, - {0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE}, - {0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4}, - {0x01D6, 0x01D6}, {0x01D8, 0x01D8}, {0x01DA, 0x01DA}, - {0x01DC, 0x01DC}, {0x0251, 0x0251}, {0x0261, 0x0261}, - {0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB}, - {0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB}, - {0x02DD, 0x02DD}, {0x02DF, 0x02DF}, {0x0300, 0x036F}, - {0x0391, 0x03A1}, {0x03A3, 0x03A9}, {0x03B1, 0x03C1}, - {0x03C3, 0x03C9}, {0x0401, 0x0401}, {0x0410, 0x044F}, - {0x0451, 0x0451}, {0x2010, 0x2010}, {0x2013, 0x2016}, - {0x2018, 0x2019}, {0x201C, 0x201D}, {0x2020, 0x2022}, - {0x2024, 0x2027}, {0x2030, 0x2030}, {0x2032, 0x2033}, - {0x2035, 0x2035}, {0x203B, 0x203B}, {0x203E, 0x203E}, - {0x2074, 0x2074}, {0x207F, 0x207F}, {0x2081, 0x2084}, - {0x20AC, 0x20AC}, {0x2103, 0x2103}, {0x2105, 0x2105}, - {0x2109, 0x2109}, {0x2113, 0x2113}, {0x2116, 0x2116}, - {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B}, - {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, - {0x2170, 0x2179}, {0x2189, 0x2189}, {0x2190, 0x2199}, - {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4}, - {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, - {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, - {0x2211, 0x2211}, {0x2215, 0x2215}, {0x221A, 0x221A}, - {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225}, - {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, - {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, - {0x2252, 0x2252}, {0x2260, 0x2261}, {0x2264, 0x2267}, - {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283}, - {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, - {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, - {0x2460, 0x24E9}, {0x24EB, 0x254B}, {0x2550, 0x2573}, - {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1}, - {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, - {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, - {0x25CB, 0x25CB}, {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, - {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609}, - {0x260E, 0x260F}, {0x261C, 0x261C}, {0x261E, 0x261E}, - {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, - {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, - {0x266F, 0x266F}, {0x269E, 0x269F}, {0x26BF, 0x26BF}, - {0x26C6, 0x26CD}, {0x26CF, 0x26D3}, {0x26D5, 0x26E1}, - {0x26E3, 0x26E3}, {0x26E8, 0x26E9}, {0x26EB, 0x26F1}, - {0x26F4, 0x26F4}, {0x26F6, 0x26F9}, {0x26FB, 0x26FC}, - {0x26FE, 0x26FF}, {0x273D, 0x273D}, {0x2776, 0x277F}, - {0x2B56, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF}, - {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, - {0x1F110, 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F18D}, - {0x1F18F, 0x1F190}, {0x1F19B, 0x1F1AC}, {0xE0100, 0xE01EF}, - {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, -} - -var emoji = table{ - {0x203C, 0x203C}, {0x2049, 0x2049}, {0x2122, 0x2122}, - {0x2139, 0x2139}, {0x2194, 0x2199}, {0x21A9, 0x21AA}, - {0x231A, 0x231B}, {0x2328, 0x2328}, {0x23CF, 0x23CF}, - {0x23E9, 0x23F3}, {0x23F8, 0x23FA}, {0x24C2, 0x24C2}, - {0x25AA, 0x25AB}, {0x25B6, 0x25B6}, {0x25C0, 0x25C0}, - {0x25FB, 0x25FE}, {0x2600, 0x2604}, {0x260E, 0x260E}, - {0x2611, 0x2611}, {0x2614, 0x2615}, {0x2618, 0x2618}, - {0x261D, 0x261D}, {0x2620, 0x2620}, {0x2622, 0x2623}, - {0x2626, 0x2626}, {0x262A, 0x262A}, {0x262E, 0x262F}, - {0x2638, 0x263A}, {0x2640, 0x2640}, {0x2642, 0x2642}, - {0x2648, 0x2653}, {0x265F, 0x2660}, {0x2663, 0x2663}, - {0x2665, 0x2666}, {0x2668, 0x2668}, {0x267B, 0x267B}, - {0x267E, 0x267F}, {0x2692, 0x2697}, {0x2699, 0x2699}, - {0x269B, 0x269C}, {0x26A0, 0x26A1}, {0x26AA, 0x26AB}, - {0x26B0, 0x26B1}, {0x26BD, 0x26BE}, {0x26C4, 0x26C5}, - {0x26C8, 0x26C8}, {0x26CE, 0x26CF}, {0x26D1, 0x26D1}, - {0x26D3, 0x26D4}, {0x26E9, 0x26EA}, {0x26F0, 0x26F5}, - {0x26F7, 0x26FA}, {0x26FD, 0x26FD}, {0x2702, 0x2702}, - {0x2705, 0x2705}, {0x2708, 0x270D}, {0x270F, 0x270F}, - {0x2712, 0x2712}, {0x2714, 0x2714}, {0x2716, 0x2716}, - {0x271D, 0x271D}, {0x2721, 0x2721}, {0x2728, 0x2728}, - {0x2733, 0x2734}, {0x2744, 0x2744}, {0x2747, 0x2747}, - {0x274C, 0x274C}, {0x274E, 0x274E}, {0x2753, 0x2755}, - {0x2757, 0x2757}, {0x2763, 0x2764}, {0x2795, 0x2797}, - {0x27A1, 0x27A1}, {0x27B0, 0x27B0}, {0x27BF, 0x27BF}, - {0x2934, 0x2935}, {0x2B05, 0x2B07}, {0x2B1B, 0x2B1C}, - {0x2B50, 0x2B50}, {0x2B55, 0x2B55}, {0x3030, 0x3030}, - {0x303D, 0x303D}, {0x3297, 0x3297}, {0x3299, 0x3299}, - {0x1F004, 0x1F004}, {0x1F0CF, 0x1F0CF}, {0x1F170, 0x1F171}, - {0x1F17E, 0x1F17F}, {0x1F18E, 0x1F18E}, {0x1F191, 0x1F19A}, - {0x1F1E6, 0x1F1FF}, {0x1F201, 0x1F202}, {0x1F21A, 0x1F21A}, - {0x1F22F, 0x1F22F}, {0x1F232, 0x1F23A}, {0x1F250, 0x1F251}, - {0x1F300, 0x1F321}, {0x1F324, 0x1F393}, {0x1F396, 0x1F397}, - {0x1F399, 0x1F39B}, {0x1F39E, 0x1F3F0}, {0x1F3F3, 0x1F3F5}, - {0x1F3F7, 0x1F4FD}, {0x1F4FF, 0x1F53D}, {0x1F549, 0x1F54E}, - {0x1F550, 0x1F567}, {0x1F56F, 0x1F570}, {0x1F573, 0x1F57A}, - {0x1F587, 0x1F587}, {0x1F58A, 0x1F58D}, {0x1F590, 0x1F590}, - {0x1F595, 0x1F596}, {0x1F5A4, 0x1F5A5}, {0x1F5A8, 0x1F5A8}, - {0x1F5B1, 0x1F5B2}, {0x1F5BC, 0x1F5BC}, {0x1F5C2, 0x1F5C4}, - {0x1F5D1, 0x1F5D3}, {0x1F5DC, 0x1F5DE}, {0x1F5E1, 0x1F5E1}, - {0x1F5E3, 0x1F5E3}, {0x1F5E8, 0x1F5E8}, {0x1F5EF, 0x1F5EF}, - {0x1F5F3, 0x1F5F3}, {0x1F5FA, 0x1F64F}, {0x1F680, 0x1F6C5}, - {0x1F6CB, 0x1F6D2}, {0x1F6E0, 0x1F6E5}, {0x1F6E9, 0x1F6E9}, - {0x1F6EB, 0x1F6EC}, {0x1F6F0, 0x1F6F0}, {0x1F6F3, 0x1F6F9}, - {0x1F910, 0x1F93A}, {0x1F93C, 0x1F93E}, {0x1F940, 0x1F945}, - {0x1F947, 0x1F970}, {0x1F973, 0x1F976}, {0x1F97A, 0x1F97A}, - {0x1F97C, 0x1F9A2}, {0x1F9B0, 0x1F9B9}, {0x1F9C0, 0x1F9C2}, - {0x1F9D0, 0x1F9FF}, -} - -var notassigned = table{ - {0x0378, 0x0379}, {0x0380, 0x0383}, {0x038B, 0x038B}, - {0x038D, 0x038D}, {0x03A2, 0x03A2}, {0x0530, 0x0530}, - {0x0557, 0x0558}, {0x0560, 0x0560}, {0x0588, 0x0588}, - {0x058B, 0x058C}, {0x0590, 0x0590}, {0x05C8, 0x05CF}, - {0x05EB, 0x05EF}, {0x05F5, 0x05FF}, {0x061D, 0x061D}, - {0x070E, 0x070E}, {0x074B, 0x074C}, {0x07B2, 0x07BF}, - {0x07FB, 0x07FF}, {0x082E, 0x082F}, {0x083F, 0x083F}, - {0x085C, 0x085D}, {0x085F, 0x089F}, {0x08B5, 0x08B5}, - {0x08BE, 0x08D3}, {0x0984, 0x0984}, {0x098D, 0x098E}, - {0x0991, 0x0992}, {0x09A9, 0x09A9}, {0x09B1, 0x09B1}, - {0x09B3, 0x09B5}, {0x09BA, 0x09BB}, {0x09C5, 0x09C6}, - {0x09C9, 0x09CA}, {0x09CF, 0x09D6}, {0x09D8, 0x09DB}, - {0x09DE, 0x09DE}, {0x09E4, 0x09E5}, {0x09FC, 0x0A00}, - {0x0A04, 0x0A04}, {0x0A0B, 0x0A0E}, {0x0A11, 0x0A12}, - {0x0A29, 0x0A29}, {0x0A31, 0x0A31}, {0x0A34, 0x0A34}, - {0x0A37, 0x0A37}, {0x0A3A, 0x0A3B}, {0x0A3D, 0x0A3D}, - {0x0A43, 0x0A46}, {0x0A49, 0x0A4A}, {0x0A4E, 0x0A50}, - {0x0A52, 0x0A58}, {0x0A5D, 0x0A5D}, {0x0A5F, 0x0A65}, - {0x0A76, 0x0A80}, {0x0A84, 0x0A84}, {0x0A8E, 0x0A8E}, - {0x0A92, 0x0A92}, {0x0AA9, 0x0AA9}, {0x0AB1, 0x0AB1}, - {0x0AB4, 0x0AB4}, {0x0ABA, 0x0ABB}, {0x0AC6, 0x0AC6}, - {0x0ACA, 0x0ACA}, {0x0ACE, 0x0ACF}, {0x0AD1, 0x0ADF}, - {0x0AE4, 0x0AE5}, {0x0AF2, 0x0AF8}, {0x0AFA, 0x0B00}, - {0x0B04, 0x0B04}, {0x0B0D, 0x0B0E}, {0x0B11, 0x0B12}, - {0x0B29, 0x0B29}, {0x0B31, 0x0B31}, {0x0B34, 0x0B34}, - {0x0B3A, 0x0B3B}, {0x0B45, 0x0B46}, {0x0B49, 0x0B4A}, - {0x0B4E, 0x0B55}, {0x0B58, 0x0B5B}, {0x0B5E, 0x0B5E}, - {0x0B64, 0x0B65}, {0x0B78, 0x0B81}, {0x0B84, 0x0B84}, - {0x0B8B, 0x0B8D}, {0x0B91, 0x0B91}, {0x0B96, 0x0B98}, - {0x0B9B, 0x0B9B}, {0x0B9D, 0x0B9D}, {0x0BA0, 0x0BA2}, - {0x0BA5, 0x0BA7}, {0x0BAB, 0x0BAD}, {0x0BBA, 0x0BBD}, - {0x0BC3, 0x0BC5}, {0x0BC9, 0x0BC9}, {0x0BCE, 0x0BCF}, - {0x0BD1, 0x0BD6}, {0x0BD8, 0x0BE5}, {0x0BFB, 0x0BFF}, - {0x0C04, 0x0C04}, {0x0C0D, 0x0C0D}, {0x0C11, 0x0C11}, - {0x0C29, 0x0C29}, {0x0C3A, 0x0C3C}, {0x0C45, 0x0C45}, - {0x0C49, 0x0C49}, {0x0C4E, 0x0C54}, {0x0C57, 0x0C57}, - {0x0C5B, 0x0C5F}, {0x0C64, 0x0C65}, {0x0C70, 0x0C77}, - {0x0C84, 0x0C84}, {0x0C8D, 0x0C8D}, {0x0C91, 0x0C91}, - {0x0CA9, 0x0CA9}, {0x0CB4, 0x0CB4}, {0x0CBA, 0x0CBB}, - {0x0CC5, 0x0CC5}, {0x0CC9, 0x0CC9}, {0x0CCE, 0x0CD4}, - {0x0CD7, 0x0CDD}, {0x0CDF, 0x0CDF}, {0x0CE4, 0x0CE5}, - {0x0CF0, 0x0CF0}, {0x0CF3, 0x0D00}, {0x0D04, 0x0D04}, - {0x0D0D, 0x0D0D}, {0x0D11, 0x0D11}, {0x0D3B, 0x0D3C}, - {0x0D45, 0x0D45}, {0x0D49, 0x0D49}, {0x0D50, 0x0D53}, - {0x0D64, 0x0D65}, {0x0D80, 0x0D81}, {0x0D84, 0x0D84}, - {0x0D97, 0x0D99}, {0x0DB2, 0x0DB2}, {0x0DBC, 0x0DBC}, - {0x0DBE, 0x0DBF}, {0x0DC7, 0x0DC9}, {0x0DCB, 0x0DCE}, - {0x0DD5, 0x0DD5}, {0x0DD7, 0x0DD7}, {0x0DE0, 0x0DE5}, - {0x0DF0, 0x0DF1}, {0x0DF5, 0x0E00}, {0x0E3B, 0x0E3E}, - {0x0E5C, 0x0E80}, {0x0E83, 0x0E83}, {0x0E85, 0x0E86}, - {0x0E89, 0x0E89}, {0x0E8B, 0x0E8C}, {0x0E8E, 0x0E93}, - {0x0E98, 0x0E98}, {0x0EA0, 0x0EA0}, {0x0EA4, 0x0EA4}, - {0x0EA6, 0x0EA6}, {0x0EA8, 0x0EA9}, {0x0EAC, 0x0EAC}, - {0x0EBA, 0x0EBA}, {0x0EBE, 0x0EBF}, {0x0EC5, 0x0EC5}, - {0x0EC7, 0x0EC7}, {0x0ECE, 0x0ECF}, {0x0EDA, 0x0EDB}, - {0x0EE0, 0x0EFF}, {0x0F48, 0x0F48}, {0x0F6D, 0x0F70}, - {0x0F98, 0x0F98}, {0x0FBD, 0x0FBD}, {0x0FCD, 0x0FCD}, - {0x0FDB, 0x0FFF}, {0x10C6, 0x10C6}, {0x10C8, 0x10CC}, - {0x10CE, 0x10CF}, {0x1249, 0x1249}, {0x124E, 0x124F}, - {0x1257, 0x1257}, {0x1259, 0x1259}, {0x125E, 0x125F}, - {0x1289, 0x1289}, {0x128E, 0x128F}, {0x12B1, 0x12B1}, - {0x12B6, 0x12B7}, {0x12BF, 0x12BF}, {0x12C1, 0x12C1}, - {0x12C6, 0x12C7}, {0x12D7, 0x12D7}, {0x1311, 0x1311}, - {0x1316, 0x1317}, {0x135B, 0x135C}, {0x137D, 0x137F}, - {0x139A, 0x139F}, {0x13F6, 0x13F7}, {0x13FE, 0x13FF}, - {0x169D, 0x169F}, {0x16F9, 0x16FF}, {0x170D, 0x170D}, - {0x1715, 0x171F}, {0x1737, 0x173F}, {0x1754, 0x175F}, - {0x176D, 0x176D}, {0x1771, 0x1771}, {0x1774, 0x177F}, - {0x17DE, 0x17DF}, {0x17EA, 0x17EF}, {0x17FA, 0x17FF}, - {0x180F, 0x180F}, {0x181A, 0x181F}, {0x1878, 0x187F}, - {0x18AB, 0x18AF}, {0x18F6, 0x18FF}, {0x191F, 0x191F}, - {0x192C, 0x192F}, {0x193C, 0x193F}, {0x1941, 0x1943}, - {0x196E, 0x196F}, {0x1975, 0x197F}, {0x19AC, 0x19AF}, - {0x19CA, 0x19CF}, {0x19DB, 0x19DD}, {0x1A1C, 0x1A1D}, - {0x1A5F, 0x1A5F}, {0x1A7D, 0x1A7E}, {0x1A8A, 0x1A8F}, - {0x1A9A, 0x1A9F}, {0x1AAE, 0x1AAF}, {0x1ABF, 0x1AFF}, - {0x1B4C, 0x1B4F}, {0x1B7D, 0x1B7F}, {0x1BF4, 0x1BFB}, - {0x1C38, 0x1C3A}, {0x1C4A, 0x1C4C}, {0x1C89, 0x1CBF}, - {0x1CC8, 0x1CCF}, {0x1CF7, 0x1CF7}, {0x1CFA, 0x1CFF}, - {0x1DF6, 0x1DFA}, {0x1F16, 0x1F17}, {0x1F1E, 0x1F1F}, - {0x1F46, 0x1F47}, {0x1F4E, 0x1F4F}, {0x1F58, 0x1F58}, - {0x1F5A, 0x1F5A}, {0x1F5C, 0x1F5C}, {0x1F5E, 0x1F5E}, - {0x1F7E, 0x1F7F}, {0x1FB5, 0x1FB5}, {0x1FC5, 0x1FC5}, - {0x1FD4, 0x1FD5}, {0x1FDC, 0x1FDC}, {0x1FF0, 0x1FF1}, - {0x1FF5, 0x1FF5}, {0x1FFF, 0x1FFF}, {0x2065, 0x2065}, - {0x2072, 0x2073}, {0x208F, 0x208F}, {0x209D, 0x209F}, - {0x20BF, 0x20CF}, {0x20F1, 0x20FF}, {0x218C, 0x218F}, - {0x23FF, 0x23FF}, {0x2427, 0x243F}, {0x244B, 0x245F}, - {0x2B74, 0x2B75}, {0x2B96, 0x2B97}, {0x2BBA, 0x2BBC}, - {0x2BC9, 0x2BC9}, {0x2BD2, 0x2BEB}, {0x2BF0, 0x2BFF}, - {0x2C2F, 0x2C2F}, {0x2C5F, 0x2C5F}, {0x2CF4, 0x2CF8}, - {0x2D26, 0x2D26}, {0x2D28, 0x2D2C}, {0x2D2E, 0x2D2F}, - {0x2D68, 0x2D6E}, {0x2D71, 0x2D7E}, {0x2D97, 0x2D9F}, - {0x2DA7, 0x2DA7}, {0x2DAF, 0x2DAF}, {0x2DB7, 0x2DB7}, - {0x2DBF, 0x2DBF}, {0x2DC7, 0x2DC7}, {0x2DCF, 0x2DCF}, - {0x2DD7, 0x2DD7}, {0x2DDF, 0x2DDF}, {0x2E45, 0x2E7F}, - {0x2E9A, 0x2E9A}, {0x2EF4, 0x2EFF}, {0x2FD6, 0x2FEF}, - {0x2FFC, 0x2FFF}, {0x3040, 0x3040}, {0x3097, 0x3098}, - {0x3100, 0x3104}, {0x312E, 0x3130}, {0x318F, 0x318F}, - {0x31BB, 0x31BF}, {0x31E4, 0x31EF}, {0x321F, 0x321F}, - {0x32FF, 0x32FF}, {0x4DB6, 0x4DBF}, {0x9FD6, 0x9FFF}, - {0xA48D, 0xA48F}, {0xA4C7, 0xA4CF}, {0xA62C, 0xA63F}, - {0xA6F8, 0xA6FF}, {0xA7AF, 0xA7AF}, {0xA7B8, 0xA7F6}, - {0xA82C, 0xA82F}, {0xA83A, 0xA83F}, {0xA878, 0xA87F}, - {0xA8C6, 0xA8CD}, {0xA8DA, 0xA8DF}, {0xA8FE, 0xA8FF}, - {0xA954, 0xA95E}, {0xA97D, 0xA97F}, {0xA9CE, 0xA9CE}, - {0xA9DA, 0xA9DD}, {0xA9FF, 0xA9FF}, {0xAA37, 0xAA3F}, - {0xAA4E, 0xAA4F}, {0xAA5A, 0xAA5B}, {0xAAC3, 0xAADA}, - {0xAAF7, 0xAB00}, {0xAB07, 0xAB08}, {0xAB0F, 0xAB10}, - {0xAB17, 0xAB1F}, {0xAB27, 0xAB27}, {0xAB2F, 0xAB2F}, - {0xAB66, 0xAB6F}, {0xABEE, 0xABEF}, {0xABFA, 0xABFF}, - {0xD7A4, 0xD7AF}, {0xD7C7, 0xD7CA}, {0xD7FC, 0xD7FF}, - {0xFA6E, 0xFA6F}, {0xFADA, 0xFAFF}, {0xFB07, 0xFB12}, - {0xFB18, 0xFB1C}, {0xFB37, 0xFB37}, {0xFB3D, 0xFB3D}, - {0xFB3F, 0xFB3F}, {0xFB42, 0xFB42}, {0xFB45, 0xFB45}, - {0xFBC2, 0xFBD2}, {0xFD40, 0xFD4F}, {0xFD90, 0xFD91}, - {0xFDC8, 0xFDEF}, {0xFDFE, 0xFDFF}, {0xFE1A, 0xFE1F}, - {0xFE53, 0xFE53}, {0xFE67, 0xFE67}, {0xFE6C, 0xFE6F}, - {0xFE75, 0xFE75}, {0xFEFD, 0xFEFE}, {0xFF00, 0xFF00}, - {0xFFBF, 0xFFC1}, {0xFFC8, 0xFFC9}, {0xFFD0, 0xFFD1}, - {0xFFD8, 0xFFD9}, {0xFFDD, 0xFFDF}, {0xFFE7, 0xFFE7}, - {0xFFEF, 0xFFF8}, {0xFFFE, 0xFFFF}, {0x1000C, 0x1000C}, - {0x10027, 0x10027}, {0x1003B, 0x1003B}, {0x1003E, 0x1003E}, - {0x1004E, 0x1004F}, {0x1005E, 0x1007F}, {0x100FB, 0x100FF}, - {0x10103, 0x10106}, {0x10134, 0x10136}, {0x1018F, 0x1018F}, - {0x1019C, 0x1019F}, {0x101A1, 0x101CF}, {0x101FE, 0x1027F}, - {0x1029D, 0x1029F}, {0x102D1, 0x102DF}, {0x102FC, 0x102FF}, - {0x10324, 0x1032F}, {0x1034B, 0x1034F}, {0x1037B, 0x1037F}, - {0x1039E, 0x1039E}, {0x103C4, 0x103C7}, {0x103D6, 0x103FF}, - {0x1049E, 0x1049F}, {0x104AA, 0x104AF}, {0x104D4, 0x104D7}, - {0x104FC, 0x104FF}, {0x10528, 0x1052F}, {0x10564, 0x1056E}, - {0x10570, 0x105FF}, {0x10737, 0x1073F}, {0x10756, 0x1075F}, - {0x10768, 0x107FF}, {0x10806, 0x10807}, {0x10809, 0x10809}, - {0x10836, 0x10836}, {0x10839, 0x1083B}, {0x1083D, 0x1083E}, - {0x10856, 0x10856}, {0x1089F, 0x108A6}, {0x108B0, 0x108DF}, - {0x108F3, 0x108F3}, {0x108F6, 0x108FA}, {0x1091C, 0x1091E}, - {0x1093A, 0x1093E}, {0x10940, 0x1097F}, {0x109B8, 0x109BB}, - {0x109D0, 0x109D1}, {0x10A04, 0x10A04}, {0x10A07, 0x10A0B}, - {0x10A14, 0x10A14}, {0x10A18, 0x10A18}, {0x10A34, 0x10A37}, - {0x10A3B, 0x10A3E}, {0x10A48, 0x10A4F}, {0x10A59, 0x10A5F}, - {0x10AA0, 0x10ABF}, {0x10AE7, 0x10AEA}, {0x10AF7, 0x10AFF}, - {0x10B36, 0x10B38}, {0x10B56, 0x10B57}, {0x10B73, 0x10B77}, - {0x10B92, 0x10B98}, {0x10B9D, 0x10BA8}, {0x10BB0, 0x10BFF}, - {0x10C49, 0x10C7F}, {0x10CB3, 0x10CBF}, {0x10CF3, 0x10CF9}, - {0x10D00, 0x10E5F}, {0x10E7F, 0x10FFF}, {0x1104E, 0x11051}, - {0x11070, 0x1107E}, {0x110C2, 0x110CF}, {0x110E9, 0x110EF}, - {0x110FA, 0x110FF}, {0x11135, 0x11135}, {0x11144, 0x1114F}, - {0x11177, 0x1117F}, {0x111CE, 0x111CF}, {0x111E0, 0x111E0}, - {0x111F5, 0x111FF}, {0x11212, 0x11212}, {0x1123F, 0x1127F}, - {0x11287, 0x11287}, {0x11289, 0x11289}, {0x1128E, 0x1128E}, - {0x1129E, 0x1129E}, {0x112AA, 0x112AF}, {0x112EB, 0x112EF}, - {0x112FA, 0x112FF}, {0x11304, 0x11304}, {0x1130D, 0x1130E}, - {0x11311, 0x11312}, {0x11329, 0x11329}, {0x11331, 0x11331}, - {0x11334, 0x11334}, {0x1133A, 0x1133B}, {0x11345, 0x11346}, - {0x11349, 0x1134A}, {0x1134E, 0x1134F}, {0x11351, 0x11356}, - {0x11358, 0x1135C}, {0x11364, 0x11365}, {0x1136D, 0x1136F}, - {0x11375, 0x113FF}, {0x1145A, 0x1145A}, {0x1145C, 0x1145C}, - {0x1145E, 0x1147F}, {0x114C8, 0x114CF}, {0x114DA, 0x1157F}, - {0x115B6, 0x115B7}, {0x115DE, 0x115FF}, {0x11645, 0x1164F}, - {0x1165A, 0x1165F}, {0x1166D, 0x1167F}, {0x116B8, 0x116BF}, - {0x116CA, 0x116FF}, {0x1171A, 0x1171C}, {0x1172C, 0x1172F}, - {0x11740, 0x1189F}, {0x118F3, 0x118FE}, {0x11900, 0x11ABF}, - {0x11AF9, 0x11BFF}, {0x11C09, 0x11C09}, {0x11C37, 0x11C37}, - {0x11C46, 0x11C4F}, {0x11C6D, 0x11C6F}, {0x11C90, 0x11C91}, - {0x11CA8, 0x11CA8}, {0x11CB7, 0x11FFF}, {0x1239A, 0x123FF}, - {0x1246F, 0x1246F}, {0x12475, 0x1247F}, {0x12544, 0x12FFF}, - {0x1342F, 0x143FF}, {0x14647, 0x167FF}, {0x16A39, 0x16A3F}, - {0x16A5F, 0x16A5F}, {0x16A6A, 0x16A6D}, {0x16A70, 0x16ACF}, - {0x16AEE, 0x16AEF}, {0x16AF6, 0x16AFF}, {0x16B46, 0x16B4F}, - {0x16B5A, 0x16B5A}, {0x16B62, 0x16B62}, {0x16B78, 0x16B7C}, - {0x16B90, 0x16EFF}, {0x16F45, 0x16F4F}, {0x16F7F, 0x16F8E}, - {0x16FA0, 0x16FDF}, {0x16FE1, 0x16FFF}, {0x187ED, 0x187FF}, - {0x18AF3, 0x1AFFF}, {0x1B002, 0x1BBFF}, {0x1BC6B, 0x1BC6F}, - {0x1BC7D, 0x1BC7F}, {0x1BC89, 0x1BC8F}, {0x1BC9A, 0x1BC9B}, - {0x1BCA4, 0x1CFFF}, {0x1D0F6, 0x1D0FF}, {0x1D127, 0x1D128}, - {0x1D1E9, 0x1D1FF}, {0x1D246, 0x1D2FF}, {0x1D357, 0x1D35F}, - {0x1D372, 0x1D3FF}, {0x1D455, 0x1D455}, {0x1D49D, 0x1D49D}, - {0x1D4A0, 0x1D4A1}, {0x1D4A3, 0x1D4A4}, {0x1D4A7, 0x1D4A8}, - {0x1D4AD, 0x1D4AD}, {0x1D4BA, 0x1D4BA}, {0x1D4BC, 0x1D4BC}, - {0x1D4C4, 0x1D4C4}, {0x1D506, 0x1D506}, {0x1D50B, 0x1D50C}, - {0x1D515, 0x1D515}, {0x1D51D, 0x1D51D}, {0x1D53A, 0x1D53A}, - {0x1D53F, 0x1D53F}, {0x1D545, 0x1D545}, {0x1D547, 0x1D549}, - {0x1D551, 0x1D551}, {0x1D6A6, 0x1D6A7}, {0x1D7CC, 0x1D7CD}, - {0x1DA8C, 0x1DA9A}, {0x1DAA0, 0x1DAA0}, {0x1DAB0, 0x1DFFF}, - {0x1E007, 0x1E007}, {0x1E019, 0x1E01A}, {0x1E022, 0x1E022}, - {0x1E025, 0x1E025}, {0x1E02B, 0x1E7FF}, {0x1E8C5, 0x1E8C6}, - {0x1E8D7, 0x1E8FF}, {0x1E94B, 0x1E94F}, {0x1E95A, 0x1E95D}, - {0x1E960, 0x1EDFF}, {0x1EE04, 0x1EE04}, {0x1EE20, 0x1EE20}, - {0x1EE23, 0x1EE23}, {0x1EE25, 0x1EE26}, {0x1EE28, 0x1EE28}, - {0x1EE33, 0x1EE33}, {0x1EE38, 0x1EE38}, {0x1EE3A, 0x1EE3A}, - {0x1EE3C, 0x1EE41}, {0x1EE43, 0x1EE46}, {0x1EE48, 0x1EE48}, - {0x1EE4A, 0x1EE4A}, {0x1EE4C, 0x1EE4C}, {0x1EE50, 0x1EE50}, - {0x1EE53, 0x1EE53}, {0x1EE55, 0x1EE56}, {0x1EE58, 0x1EE58}, - {0x1EE5A, 0x1EE5A}, {0x1EE5C, 0x1EE5C}, {0x1EE5E, 0x1EE5E}, - {0x1EE60, 0x1EE60}, {0x1EE63, 0x1EE63}, {0x1EE65, 0x1EE66}, - {0x1EE6B, 0x1EE6B}, {0x1EE73, 0x1EE73}, {0x1EE78, 0x1EE78}, - {0x1EE7D, 0x1EE7D}, {0x1EE7F, 0x1EE7F}, {0x1EE8A, 0x1EE8A}, - {0x1EE9C, 0x1EEA0}, {0x1EEA4, 0x1EEA4}, {0x1EEAA, 0x1EEAA}, - {0x1EEBC, 0x1EEEF}, {0x1EEF2, 0x1EFFF}, {0x1F02C, 0x1F02F}, - {0x1F094, 0x1F09F}, {0x1F0AF, 0x1F0B0}, {0x1F0C0, 0x1F0C0}, - {0x1F0D0, 0x1F0D0}, {0x1F0F6, 0x1F0FF}, {0x1F10D, 0x1F10F}, - {0x1F12F, 0x1F12F}, {0x1F16C, 0x1F16F}, {0x1F1AD, 0x1F1E5}, - {0x1F203, 0x1F20F}, {0x1F23C, 0x1F23F}, {0x1F249, 0x1F24F}, - {0x1F252, 0x1F2FF}, {0x1F6D3, 0x1F6DF}, {0x1F6ED, 0x1F6EF}, - {0x1F6F7, 0x1F6FF}, {0x1F774, 0x1F77F}, {0x1F7D5, 0x1F7FF}, - {0x1F80C, 0x1F80F}, {0x1F848, 0x1F84F}, {0x1F85A, 0x1F85F}, - {0x1F888, 0x1F88F}, {0x1F8AE, 0x1F90F}, {0x1F91F, 0x1F91F}, - {0x1F928, 0x1F92F}, {0x1F931, 0x1F932}, {0x1F93F, 0x1F93F}, - {0x1F94C, 0x1F94F}, {0x1F95F, 0x1F97F}, {0x1F992, 0x1F9BF}, - {0x1F9C1, 0x1FFFF}, {0x2A6D7, 0x2A6FF}, {0x2B735, 0x2B73F}, - {0x2B81E, 0x2B81F}, {0x2CEA2, 0x2F7FF}, {0x2FA1E, 0xE0000}, - {0xE0002, 0xE001F}, {0xE0080, 0xE00FF}, {0xE01F0, 0xEFFFF}, - {0xFFFFE, 0xFFFFF}, -} - -var neutral = table{ - {0x0000, 0x001F}, {0x007F, 0x00A0}, {0x00A9, 0x00A9}, - {0x00AB, 0x00AB}, {0x00B5, 0x00B5}, {0x00BB, 0x00BB}, - {0x00C0, 0x00C5}, {0x00C7, 0x00CF}, {0x00D1, 0x00D6}, - {0x00D9, 0x00DD}, {0x00E2, 0x00E5}, {0x00E7, 0x00E7}, - {0x00EB, 0x00EB}, {0x00EE, 0x00EF}, {0x00F1, 0x00F1}, - {0x00F4, 0x00F6}, {0x00FB, 0x00FB}, {0x00FD, 0x00FD}, - {0x00FF, 0x0100}, {0x0102, 0x0110}, {0x0112, 0x0112}, - {0x0114, 0x011A}, {0x011C, 0x0125}, {0x0128, 0x012A}, - {0x012C, 0x0130}, {0x0134, 0x0137}, {0x0139, 0x013E}, - {0x0143, 0x0143}, {0x0145, 0x0147}, {0x014C, 0x014C}, - {0x014E, 0x0151}, {0x0154, 0x0165}, {0x0168, 0x016A}, - {0x016C, 0x01CD}, {0x01CF, 0x01CF}, {0x01D1, 0x01D1}, - {0x01D3, 0x01D3}, {0x01D5, 0x01D5}, {0x01D7, 0x01D7}, - {0x01D9, 0x01D9}, {0x01DB, 0x01DB}, {0x01DD, 0x0250}, - {0x0252, 0x0260}, {0x0262, 0x02C3}, {0x02C5, 0x02C6}, - {0x02C8, 0x02C8}, {0x02CC, 0x02CC}, {0x02CE, 0x02CF}, - {0x02D1, 0x02D7}, {0x02DC, 0x02DC}, {0x02DE, 0x02DE}, - {0x02E0, 0x02FF}, {0x0370, 0x0377}, {0x037A, 0x037F}, - {0x0384, 0x038A}, {0x038C, 0x038C}, {0x038E, 0x0390}, - {0x03AA, 0x03B0}, {0x03C2, 0x03C2}, {0x03CA, 0x0400}, - {0x0402, 0x040F}, {0x0450, 0x0450}, {0x0452, 0x052F}, - {0x0531, 0x0556}, {0x0559, 0x055F}, {0x0561, 0x0587}, - {0x0589, 0x058A}, {0x058D, 0x058F}, {0x0591, 0x05C7}, - {0x05D0, 0x05EA}, {0x05F0, 0x05F4}, {0x0600, 0x061C}, - {0x061E, 0x070D}, {0x070F, 0x074A}, {0x074D, 0x07B1}, - {0x07C0, 0x07FA}, {0x0800, 0x082D}, {0x0830, 0x083E}, - {0x0840, 0x085B}, {0x085E, 0x085E}, {0x08A0, 0x08B4}, - {0x08B6, 0x08BD}, {0x08D4, 0x0983}, {0x0985, 0x098C}, - {0x098F, 0x0990}, {0x0993, 0x09A8}, {0x09AA, 0x09B0}, - {0x09B2, 0x09B2}, {0x09B6, 0x09B9}, {0x09BC, 0x09C4}, - {0x09C7, 0x09C8}, {0x09CB, 0x09CE}, {0x09D7, 0x09D7}, - {0x09DC, 0x09DD}, {0x09DF, 0x09E3}, {0x09E6, 0x09FB}, - {0x0A01, 0x0A03}, {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10}, - {0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, {0x0A32, 0x0A33}, - {0x0A35, 0x0A36}, {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C}, - {0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, - {0x0A51, 0x0A51}, {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, - {0x0A66, 0x0A75}, {0x0A81, 0x0A83}, {0x0A85, 0x0A8D}, - {0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0}, - {0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5}, - {0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0}, - {0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AF9}, - {0x0B01, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10}, - {0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, {0x0B32, 0x0B33}, - {0x0B35, 0x0B39}, {0x0B3C, 0x0B44}, {0x0B47, 0x0B48}, - {0x0B4B, 0x0B4D}, {0x0B56, 0x0B57}, {0x0B5C, 0x0B5D}, - {0x0B5F, 0x0B63}, {0x0B66, 0x0B77}, {0x0B82, 0x0B83}, - {0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, {0x0B92, 0x0B95}, - {0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F}, - {0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9}, - {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD}, - {0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA}, - {0x0C00, 0x0C03}, {0x0C05, 0x0C0C}, {0x0C0E, 0x0C10}, - {0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, {0x0C3D, 0x0C44}, - {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, - {0x0C58, 0x0C5A}, {0x0C60, 0x0C63}, {0x0C66, 0x0C6F}, - {0x0C78, 0x0C83}, {0x0C85, 0x0C8C}, {0x0C8E, 0x0C90}, - {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, - {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, - {0x0CD5, 0x0CD6}, {0x0CDE, 0x0CDE}, {0x0CE0, 0x0CE3}, - {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF2}, {0x0D01, 0x0D03}, - {0x0D05, 0x0D0C}, {0x0D0E, 0x0D10}, {0x0D12, 0x0D3A}, - {0x0D3D, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4F}, - {0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, {0x0D82, 0x0D83}, - {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB}, - {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA}, - {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, - {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A}, - {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, {0x0E84, 0x0E84}, - {0x0E87, 0x0E88}, {0x0E8A, 0x0E8A}, {0x0E8D, 0x0E8D}, - {0x0E94, 0x0E97}, {0x0E99, 0x0E9F}, {0x0EA1, 0x0EA3}, - {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EA7}, {0x0EAA, 0x0EAB}, - {0x0EAD, 0x0EB9}, {0x0EBB, 0x0EBD}, {0x0EC0, 0x0EC4}, - {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECD}, {0x0ED0, 0x0ED9}, - {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, - {0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, - {0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7}, - {0x10CD, 0x10CD}, {0x10D0, 0x10FF}, {0x1160, 0x1248}, - {0x124A, 0x124D}, {0x1250, 0x1256}, {0x1258, 0x1258}, - {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, - {0x1290, 0x12B0}, {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, - {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, - {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, - {0x135D, 0x137C}, {0x1380, 0x1399}, {0x13A0, 0x13F5}, - {0x13F8, 0x13FD}, {0x1400, 0x169C}, {0x16A0, 0x16F8}, - {0x1700, 0x170C}, {0x170E, 0x1714}, {0x1720, 0x1736}, - {0x1740, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770}, - {0x1772, 0x1773}, {0x1780, 0x17DD}, {0x17E0, 0x17E9}, - {0x17F0, 0x17F9}, {0x1800, 0x180E}, {0x1810, 0x1819}, - {0x1820, 0x1877}, {0x1880, 0x18AA}, {0x18B0, 0x18F5}, - {0x1900, 0x191E}, {0x1920, 0x192B}, {0x1930, 0x193B}, - {0x1940, 0x1940}, {0x1944, 0x196D}, {0x1970, 0x1974}, - {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, - {0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C}, - {0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA0, 0x1AAD}, - {0x1AB0, 0x1ABE}, {0x1B00, 0x1B4B}, {0x1B50, 0x1B7C}, - {0x1B80, 0x1BF3}, {0x1BFC, 0x1C37}, {0x1C3B, 0x1C49}, - {0x1C4D, 0x1C88}, {0x1CC0, 0x1CC7}, {0x1CD0, 0x1CF6}, - {0x1CF8, 0x1CF9}, {0x1D00, 0x1DF5}, {0x1DFB, 0x1F15}, - {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, - {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, - {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, - {0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB}, - {0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE}, - {0x2000, 0x200F}, {0x2011, 0x2012}, {0x2017, 0x2017}, - {0x201A, 0x201B}, {0x201E, 0x201F}, {0x2023, 0x2023}, - {0x2028, 0x202F}, {0x2031, 0x2031}, {0x2034, 0x2034}, - {0x2036, 0x203A}, {0x203C, 0x203D}, {0x203F, 0x2064}, - {0x2066, 0x2071}, {0x2075, 0x207E}, {0x2080, 0x2080}, - {0x2085, 0x208E}, {0x2090, 0x209C}, {0x20A0, 0x20A8}, - {0x20AA, 0x20AB}, {0x20AD, 0x20BE}, {0x20D0, 0x20F0}, - {0x2100, 0x2102}, {0x2104, 0x2104}, {0x2106, 0x2108}, - {0x210A, 0x2112}, {0x2114, 0x2115}, {0x2117, 0x2120}, - {0x2123, 0x2125}, {0x2127, 0x212A}, {0x212C, 0x2152}, - {0x2155, 0x215A}, {0x215F, 0x215F}, {0x216C, 0x216F}, - {0x217A, 0x2188}, {0x218A, 0x218B}, {0x219A, 0x21B7}, - {0x21BA, 0x21D1}, {0x21D3, 0x21D3}, {0x21D5, 0x21E6}, - {0x21E8, 0x21FF}, {0x2201, 0x2201}, {0x2204, 0x2206}, - {0x2209, 0x220A}, {0x220C, 0x220E}, {0x2210, 0x2210}, - {0x2212, 0x2214}, {0x2216, 0x2219}, {0x221B, 0x221C}, - {0x2221, 0x2222}, {0x2224, 0x2224}, {0x2226, 0x2226}, - {0x222D, 0x222D}, {0x222F, 0x2233}, {0x2238, 0x223B}, - {0x223E, 0x2247}, {0x2249, 0x224B}, {0x224D, 0x2251}, - {0x2253, 0x225F}, {0x2262, 0x2263}, {0x2268, 0x2269}, - {0x226C, 0x226D}, {0x2270, 0x2281}, {0x2284, 0x2285}, - {0x2288, 0x2294}, {0x2296, 0x2298}, {0x229A, 0x22A4}, - {0x22A6, 0x22BE}, {0x22C0, 0x2311}, {0x2313, 0x2319}, - {0x231C, 0x2328}, {0x232B, 0x23E8}, {0x23ED, 0x23EF}, - {0x23F1, 0x23F2}, {0x23F4, 0x23FE}, {0x2400, 0x2426}, - {0x2440, 0x244A}, {0x24EA, 0x24EA}, {0x254C, 0x254F}, - {0x2574, 0x257F}, {0x2590, 0x2591}, {0x2596, 0x259F}, - {0x25A2, 0x25A2}, {0x25AA, 0x25B1}, {0x25B4, 0x25B5}, - {0x25B8, 0x25BB}, {0x25BE, 0x25BF}, {0x25C2, 0x25C5}, - {0x25C9, 0x25CA}, {0x25CC, 0x25CD}, {0x25D2, 0x25E1}, - {0x25E6, 0x25EE}, {0x25F0, 0x25FC}, {0x25FF, 0x2604}, - {0x2607, 0x2608}, {0x260A, 0x260D}, {0x2610, 0x2613}, - {0x2616, 0x261B}, {0x261D, 0x261D}, {0x261F, 0x263F}, - {0x2641, 0x2641}, {0x2643, 0x2647}, {0x2654, 0x265F}, - {0x2662, 0x2662}, {0x2666, 0x2666}, {0x266B, 0x266B}, - {0x266E, 0x266E}, {0x2670, 0x267E}, {0x2680, 0x2692}, - {0x2694, 0x269D}, {0x26A0, 0x26A0}, {0x26A2, 0x26A9}, - {0x26AC, 0x26BC}, {0x26C0, 0x26C3}, {0x26E2, 0x26E2}, - {0x26E4, 0x26E7}, {0x2700, 0x2704}, {0x2706, 0x2709}, - {0x270C, 0x2727}, {0x2729, 0x273C}, {0x273E, 0x274B}, - {0x274D, 0x274D}, {0x274F, 0x2752}, {0x2756, 0x2756}, - {0x2758, 0x2775}, {0x2780, 0x2794}, {0x2798, 0x27AF}, - {0x27B1, 0x27BE}, {0x27C0, 0x27E5}, {0x27EE, 0x2984}, - {0x2987, 0x2B1A}, {0x2B1D, 0x2B4F}, {0x2B51, 0x2B54}, - {0x2B5A, 0x2B73}, {0x2B76, 0x2B95}, {0x2B98, 0x2BB9}, - {0x2BBD, 0x2BC8}, {0x2BCA, 0x2BD1}, {0x2BEC, 0x2BEF}, - {0x2C00, 0x2C2E}, {0x2C30, 0x2C5E}, {0x2C60, 0x2CF3}, - {0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, - {0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, {0x2D7F, 0x2D96}, - {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, - {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, {0x2DC8, 0x2DCE}, - {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2DE0, 0x2E44}, - {0x303F, 0x303F}, {0x4DC0, 0x4DFF}, {0xA4D0, 0xA62B}, - {0xA640, 0xA6F7}, {0xA700, 0xA7AE}, {0xA7B0, 0xA7B7}, - {0xA7F7, 0xA82B}, {0xA830, 0xA839}, {0xA840, 0xA877}, - {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, {0xA8E0, 0xA8FD}, - {0xA900, 0xA953}, {0xA95F, 0xA95F}, {0xA980, 0xA9CD}, - {0xA9CF, 0xA9D9}, {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36}, - {0xAA40, 0xAA4D}, {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2}, - {0xAADB, 0xAAF6}, {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, - {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, - {0xAB30, 0xAB65}, {0xAB70, 0xABED}, {0xABF0, 0xABF9}, - {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xD800, 0xDFFF}, - {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, {0xFB1D, 0xFB36}, - {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, {0xFB40, 0xFB41}, - {0xFB43, 0xFB44}, {0xFB46, 0xFBC1}, {0xFBD3, 0xFD3F}, - {0xFD50, 0xFD8F}, {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFD}, - {0xFE20, 0xFE2F}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, - {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFC}, {0x10000, 0x1000B}, - {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, - {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, - {0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E}, - {0x10190, 0x1019B}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD}, - {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB}, - {0x10300, 0x10323}, {0x10330, 0x1034A}, {0x10350, 0x1037A}, - {0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5}, - {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, - {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, - {0x1056F, 0x1056F}, {0x10600, 0x10736}, {0x10740, 0x10755}, - {0x10760, 0x10767}, {0x10800, 0x10805}, {0x10808, 0x10808}, - {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, - {0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF}, - {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, - {0x1091F, 0x10939}, {0x1093F, 0x1093F}, {0x10980, 0x109B7}, - {0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06}, - {0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A33}, - {0x10A38, 0x10A3A}, {0x10A3F, 0x10A47}, {0x10A50, 0x10A58}, - {0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6}, - {0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72}, - {0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, - {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, - {0x10CFA, 0x10CFF}, {0x10E60, 0x10E7E}, {0x11000, 0x1104D}, - {0x11052, 0x1106F}, {0x1107F, 0x110C1}, {0x110D0, 0x110E8}, - {0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11143}, - {0x11150, 0x11176}, {0x11180, 0x111CD}, {0x111D0, 0x111DF}, - {0x111E1, 0x111F4}, {0x11200, 0x11211}, {0x11213, 0x1123E}, - {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, - {0x1128F, 0x1129D}, {0x1129F, 0x112A9}, {0x112B0, 0x112EA}, - {0x112F0, 0x112F9}, {0x11300, 0x11303}, {0x11305, 0x1130C}, - {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, - {0x11332, 0x11333}, {0x11335, 0x11339}, {0x1133C, 0x11344}, - {0x11347, 0x11348}, {0x1134B, 0x1134D}, {0x11350, 0x11350}, - {0x11357, 0x11357}, {0x1135D, 0x11363}, {0x11366, 0x1136C}, - {0x11370, 0x11374}, {0x11400, 0x11459}, {0x1145B, 0x1145B}, - {0x1145D, 0x1145D}, {0x11480, 0x114C7}, {0x114D0, 0x114D9}, - {0x11580, 0x115B5}, {0x115B8, 0x115DD}, {0x11600, 0x11644}, - {0x11650, 0x11659}, {0x11660, 0x1166C}, {0x11680, 0x116B7}, - {0x116C0, 0x116C9}, {0x11700, 0x11719}, {0x1171D, 0x1172B}, - {0x11730, 0x1173F}, {0x118A0, 0x118F2}, {0x118FF, 0x118FF}, - {0x11AC0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, - {0x11C38, 0x11C45}, {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, - {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x12000, 0x12399}, - {0x12400, 0x1246E}, {0x12470, 0x12474}, {0x12480, 0x12543}, - {0x13000, 0x1342E}, {0x14400, 0x14646}, {0x16800, 0x16A38}, - {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16A6F}, - {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, {0x16B00, 0x16B45}, - {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, - {0x16B7D, 0x16B8F}, {0x16F00, 0x16F44}, {0x16F50, 0x16F7E}, - {0x16F8F, 0x16F9F}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, - {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3}, - {0x1D000, 0x1D0F5}, {0x1D100, 0x1D126}, {0x1D129, 0x1D1E8}, - {0x1D200, 0x1D245}, {0x1D300, 0x1D356}, {0x1D360, 0x1D371}, - {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, - {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, - {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, - {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, - {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, - {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, - {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, - {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1E000, 0x1E006}, - {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, - {0x1E026, 0x1E02A}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, - {0x1E900, 0x1E94A}, {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, - {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, - {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, - {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, - {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, - {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, - {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, - {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, - {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, - {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, - {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, - {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, - {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F003}, {0x1F005, 0x1F02B}, - {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, - {0x1F0C1, 0x1F0CE}, {0x1F0D1, 0x1F0F5}, {0x1F10B, 0x1F10C}, - {0x1F12E, 0x1F12E}, {0x1F16A, 0x1F16B}, {0x1F1E6, 0x1F1FF}, - {0x1F321, 0x1F32C}, {0x1F336, 0x1F336}, {0x1F37D, 0x1F37D}, - {0x1F394, 0x1F39F}, {0x1F3CB, 0x1F3CE}, {0x1F3D4, 0x1F3DF}, - {0x1F3F1, 0x1F3F3}, {0x1F3F5, 0x1F3F7}, {0x1F43F, 0x1F43F}, - {0x1F441, 0x1F441}, {0x1F4FD, 0x1F4FE}, {0x1F53E, 0x1F54A}, - {0x1F54F, 0x1F54F}, {0x1F568, 0x1F579}, {0x1F57B, 0x1F594}, - {0x1F597, 0x1F5A3}, {0x1F5A5, 0x1F5FA}, {0x1F650, 0x1F67F}, - {0x1F6C6, 0x1F6CB}, {0x1F6CD, 0x1F6CF}, {0x1F6E0, 0x1F6EA}, - {0x1F6F0, 0x1F6F3}, {0x1F700, 0x1F773}, {0x1F780, 0x1F7D4}, - {0x1F800, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, - {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0xE0001, 0xE0001}, - {0xE0020, 0xE007F}, -} - -// Condition have flag EastAsianWidth whether the current locale is CJK or not. -type Condition struct { - EastAsianWidth bool - ZeroWidthJoiner bool -} - -// NewCondition return new instance of Condition which is current locale. -func NewCondition() *Condition { - return &Condition{ - EastAsianWidth: EastAsianWidth, - ZeroWidthJoiner: ZeroWidthJoiner, - } -} - -// RuneWidth returns the number of cells in r. -// See http://www.unicode.org/reports/tr11/ -func (c *Condition) RuneWidth(r rune) int { - switch { - case r < 0 || r > 0x10FFFF || - inTables(r, nonprint, combining, notassigned): - return 0 - case (c.EastAsianWidth && IsAmbiguousWidth(r)) || - inTables(r, doublewidth, emoji): - return 2 - default: - return 1 - } -} - -func (c *Condition) stringWidth(s string) (width int) { - for _, r := range []rune(s) { - width += c.RuneWidth(r) - } - return width -} - -func (c *Condition) stringWidthZeroJoiner(s string) (width int) { - r1, r2 := rune(0), rune(0) - for _, r := range []rune(s) { - if r == 0xFE0E || r == 0xFE0F { - continue - } - w := c.RuneWidth(r) - if r2 == 0x200D && inTables(r, emoji) && inTables(r1, emoji) { - w = 0 - } - width += w - r1, r2 = r2, r - } - return width -} - -// StringWidth return width as you can see -func (c *Condition) StringWidth(s string) (width int) { - if c.ZeroWidthJoiner { - return c.stringWidthZeroJoiner(s) - } - return c.stringWidth(s) -} - -// Truncate return string truncated with w cells -func (c *Condition) Truncate(s string, w int, tail string) string { - if c.StringWidth(s) <= w { - return s - } - r := []rune(s) - tw := c.StringWidth(tail) - w -= tw - width := 0 - i := 0 - for ; i < len(r); i++ { - cw := c.RuneWidth(r[i]) - if width+cw > w { - break - } - width += cw - } - return string(r[0:i]) + tail -} - -// Wrap return string wrapped with w cells -func (c *Condition) Wrap(s string, w int) string { - width := 0 - out := "" - for _, r := range []rune(s) { - cw := RuneWidth(r) - if r == '\n' { - out += string(r) - width = 0 - continue - } else if width+cw > w { - out += "\n" - width = 0 - out += string(r) - width += cw - continue - } - out += string(r) - width += cw - } - return out -} - -// FillLeft return string filled in left by spaces in w cells -func (c *Condition) FillLeft(s string, w int) string { - width := c.StringWidth(s) - count := w - width - if count > 0 { - b := make([]byte, count) - for i := range b { - b[i] = ' ' - } - return string(b) + s - } - return s -} - -// FillRight return string filled in left by spaces in w cells -func (c *Condition) FillRight(s string, w int) string { - width := c.StringWidth(s) - count := w - width - if count > 0 { - b := make([]byte, count) - for i := range b { - b[i] = ' ' - } - return s + string(b) - } - return s -} - -// RuneWidth returns the number of cells in r. -// See http://www.unicode.org/reports/tr11/ -func RuneWidth(r rune) int { - return DefaultCondition.RuneWidth(r) -} - -// IsAmbiguousWidth returns whether is ambiguous width or not. -func IsAmbiguousWidth(r rune) bool { - return inTables(r, private, ambiguous) -} - -// IsNeutralWidth returns whether is neutral width or not. -func IsNeutralWidth(r rune) bool { - return inTable(r, neutral) -} - -// StringWidth return width as you can see -func StringWidth(s string) (width int) { - return DefaultCondition.StringWidth(s) -} - -// Truncate return string truncated with w cells -func Truncate(s string, w int, tail string) string { - return DefaultCondition.Truncate(s, w, tail) -} - -// Wrap return string wrapped with w cells -func Wrap(s string, w int) string { - return DefaultCondition.Wrap(s, w) -} - -// FillLeft return string filled in left by spaces in w cells -func FillLeft(s string, w int) string { - return DefaultCondition.FillLeft(s, w) -} - -// FillRight return string filled in left by spaces in w cells -func FillRight(s string, w int) string { - return DefaultCondition.FillRight(s, w) -} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go b/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go deleted file mode 100644 index 7d99f6e52..000000000 --- a/vendor/github.com/mattn/go-runewidth/runewidth_appengine.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build appengine - -package runewidth - -// IsEastAsian return true if the current locale is CJK -func IsEastAsian() bool { - return false -} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_js.go b/vendor/github.com/mattn/go-runewidth/runewidth_js.go deleted file mode 100644 index c5fdf40ba..000000000 --- a/vendor/github.com/mattn/go-runewidth/runewidth_js.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build js -// +build !appengine - -package runewidth - -func IsEastAsian() bool { - // TODO: Implement this for the web. Detect east asian in a compatible way, and return true. - return false -} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_posix.go b/vendor/github.com/mattn/go-runewidth/runewidth_posix.go deleted file mode 100644 index 66a58b5d8..000000000 --- a/vendor/github.com/mattn/go-runewidth/runewidth_posix.go +++ /dev/null @@ -1,79 +0,0 @@ -// +build !windows -// +build !js -// +build !appengine - -package runewidth - -import ( - "os" - "regexp" - "strings" -) - -var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`) - -var mblenTable = map[string]int{ - "utf-8": 6, - "utf8": 6, - "jis": 8, - "eucjp": 3, - "euckr": 2, - "euccn": 2, - "sjis": 2, - "cp932": 2, - "cp51932": 2, - "cp936": 2, - "cp949": 2, - "cp950": 2, - "big5": 2, - "gbk": 2, - "gb2312": 2, -} - -func isEastAsian(locale string) bool { - charset := strings.ToLower(locale) - r := reLoc.FindStringSubmatch(locale) - if len(r) == 2 { - charset = strings.ToLower(r[1]) - } - - if strings.HasSuffix(charset, "@cjk_narrow") { - return false - } - - for pos, b := range []byte(charset) { - if b == '@' { - charset = charset[:pos] - break - } - } - max := 1 - if m, ok := mblenTable[charset]; ok { - max = m - } - if max > 1 && (charset[0] != 'u' || - strings.HasPrefix(locale, "ja") || - strings.HasPrefix(locale, "ko") || - strings.HasPrefix(locale, "zh")) { - return true - } - return false -} - -// IsEastAsian return true if the current locale is CJK -func IsEastAsian() bool { - locale := os.Getenv("LC_CTYPE") - if locale == "" { - locale = os.Getenv("LANG") - } - - // ignore C locale - if locale == "POSIX" || locale == "C" { - return false - } - if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') { - return false - } - - return isEastAsian(locale) -} diff --git a/vendor/github.com/mattn/go-runewidth/runewidth_windows.go b/vendor/github.com/mattn/go-runewidth/runewidth_windows.go deleted file mode 100644 index d6a61777d..000000000 --- a/vendor/github.com/mattn/go-runewidth/runewidth_windows.go +++ /dev/null @@ -1,28 +0,0 @@ -// +build windows -// +build !appengine - -package runewidth - -import ( - "syscall" -) - -var ( - kernel32 = syscall.NewLazyDLL("kernel32") - procGetConsoleOutputCP = kernel32.NewProc("GetConsoleOutputCP") -) - -// IsEastAsian return true if the current locale is CJK -func IsEastAsian() bool { - r1, _, _ := procGetConsoleOutputCP.Call() - if r1 == 0 { - return false - } - - switch int(r1) { - case 932, 51932, 936, 949, 950: - return true - } - - return false -} diff --git a/vendor/github.com/vbauerster/mpb/LICENSE b/vendor/github.com/vbauerster/mpb/LICENSE new file mode 100644 index 000000000..5e68ed21e --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (C) 2016-2018 Vladimir Bauer +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/vbauerster/mpb/README.md b/vendor/github.com/vbauerster/mpb/README.md new file mode 100644 index 000000000..9b760647e --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/README.md @@ -0,0 +1,117 @@ +# Multi Progress Bar + +[](https://godoc.org/github.com/vbauerster/mpb) +[](https://travis-ci.org/vbauerster/mpb) +[](https://goreportcard.com/report/github.com/vbauerster/mpb) +[](https://codecov.io/gh/vbauerster/mpb) + +**mpb** is a Go lib for rendering progress bars in terminal applications. + +## Features + +* __Multiple Bars__: Multiple progress bars are supported +* __Dynamic Total__: [Set total](https://github.com/vbauerster/mpb/issues/9#issuecomment-344448984) while bar is running +* __Dynamic Add/Remove__: Dynamically add or remove bars +* __Cancellation__: Cancel whole rendering process +* __Predefined Decorators__: Elapsed time, [ewma](https://github.com/VividCortex/ewma) based ETA, Percentage, Bytes counter +* __Decorator's width sync__: Synchronized decorator's width among multiple bars + +## Installation + +```sh +go get github.com/vbauerster/mpb +``` + +_Note:_ it is preferable to go get from github.com, rather than gopkg.in. See issue [#11](https://github.com/vbauerster/mpb/issues/11). + +## Usage + +#### [Rendering single bar](examples/singleBar/main.go) +```go + p := mpb.New( + // override default (80) width + mpb.WithWidth(64), + // override default "[=>-]" format + mpb.WithFormat("╢▌▌░╟"), + // override default 120ms refresh rate + mpb.WithRefreshRate(180*time.Millisecond), + ) + + total := 100 + name := "Single Bar:" + // adding a single bar + bar := p.AddBar(int64(total), + mpb.PrependDecorators( + // display our name with one space on the right + decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), + // replace ETA decorator with "done" message, OnComplete event + decor.OnComplete( + // ETA decorator with ewma age of 60, and width reservation of 4 + decor.EwmaETA(decor.ET_STYLE_GO, 60, decor.WC{W: 4}), "done", + ), + ), + mpb.AppendDecorators(decor.Percentage()), + ) + // simulating some work + max := 100 * time.Millisecond + for i := 0; i < total; i++ { + start := time.Now() + time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10) + // ewma based decorators require work duration measurement + bar.IncrBy(1, time.Since(start)) + } + // wait for our bar to complete and flush + p.Wait() +``` + +#### [Rendering multiple bars](examples/simple/main.go) +```go + var wg sync.WaitGroup + p := mpb.New(mpb.WithWaitGroup(&wg)) + total, numBars := 100, 3 + wg.Add(numBars) + + for i := 0; i < numBars; i++ { + name := fmt.Sprintf("Bar#%d:", i) + bar := p.AddBar(int64(total), + mpb.PrependDecorators( + // simple name decorator + decor.Name(name), + // decor.DSyncWidth bit enables column width synchronization + decor.Percentage(decor.WCSyncSpace), + ), + mpb.AppendDecorators( + // replace ETA decorator with "done" message, OnComplete event + decor.OnComplete( + // ETA decorator with ewma age of 60 + decor.EwmaETA(decor.ET_STYLE_GO, 60), "done", + ), + ), + ) + // simulating some work + go func() { + defer wg.Done() + max := 100 * time.Millisecond + for i := 0; i < total; i++ { + start := time.Now() + time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10) + // ewma based decorators require work duration measurement + bar.IncrBy(1, time.Since(start)) + } + }() + } + // wait for all bars to complete and flush + p.Wait() +``` + +#### [Dynamic total](examples/dynTotal/main.go) + + + +#### [Complex example](examples/complex/main.go) + + + +#### [Bytes counters](examples/io/single/main.go) + + diff --git a/vendor/github.com/vbauerster/mpb/bar.go b/vendor/github.com/vbauerster/mpb/bar.go new file mode 100644 index 000000000..5a506fc84 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/bar.go @@ -0,0 +1,455 @@ +package mpb + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/vbauerster/mpb/decor" + "github.com/vbauerster/mpb/internal" +) + +const ( + rLeft = iota + rFill + rTip + rEmpty + rRight +) + +const formatLen = 5 + +type barRunes [formatLen]rune + +// Bar represents a progress Bar +type Bar struct { + priority int + index int + + runningBar *Bar + cacheState *bState + operateState chan func(*bState) + int64Ch chan int64 + boolCh chan bool + frameReaderCh chan *frameReader + syncTableCh chan [][]chan int + + // done is closed by Bar's goroutine, after cacheState is written + done chan struct{} + // shutdown is closed from master Progress goroutine only + shutdown chan struct{} +} + +type ( + bState struct { + id int + width int + total int64 + current int64 + runes barRunes + trimLeftSpace bool + trimRightSpace bool + toComplete bool + removeOnComplete bool + barClearOnComplete bool + completeFlushed bool + aDecorators []decor.Decorator + pDecorators []decor.Decorator + amountReceivers []decor.AmountReceiver + shutdownListeners []decor.ShutdownListener + refill *refill + bufP, bufB, bufA *bytes.Buffer + bufNL *bytes.Buffer + panicMsg string + newLineExtendFn func(io.Writer, *decor.Statistics) + + // following options are assigned to the *Bar + priority int + runningBar *Bar + } + refill struct { + char rune + till int64 + } + frameReader struct { + io.Reader + extendedLines int + toShutdown bool + removeOnComplete bool + } +) + +func newBar(wg *sync.WaitGroup, id int, total int64, cancel <-chan struct{}, options ...BarOption) *Bar { + if total <= 0 { + total = time.Now().Unix() + } + + s := &bState{ + id: id, + priority: id, + total: total, + } + + for _, opt := range options { + if opt != nil { + opt(s) + } + } + + s.bufP = bytes.NewBuffer(make([]byte, 0, s.width)) + s.bufB = bytes.NewBuffer(make([]byte, 0, s.width)) + s.bufA = bytes.NewBuffer(make([]byte, 0, s.width)) + + b := &Bar{ + priority: s.priority, + runningBar: s.runningBar, + operateState: make(chan func(*bState)), + int64Ch: make(chan int64), + boolCh: make(chan bool), + frameReaderCh: make(chan *frameReader, 1), + syncTableCh: make(chan [][]chan int), + done: make(chan struct{}), + shutdown: make(chan struct{}), + } + + if b.runningBar != nil { + b.priority = b.runningBar.priority + } + + if s.newLineExtendFn != nil { + s.bufNL = bytes.NewBuffer(make([]byte, 0, s.width)) + } + + go b.serve(wg, s, cancel) + return b +} + +// RemoveAllPrependers removes all prepend functions. +func (b *Bar) RemoveAllPrependers() { + select { + case b.operateState <- func(s *bState) { s.pDecorators = nil }: + case <-b.done: + } +} + +// RemoveAllAppenders removes all append functions. +func (b *Bar) RemoveAllAppenders() { + select { + case b.operateState <- func(s *bState) { s.aDecorators = nil }: + case <-b.done: + } +} + +// ProxyReader wraps r with metrics required for progress tracking. +func (b *Bar) ProxyReader(r io.Reader) io.ReadCloser { + if r == nil { + panic("expect io.Reader, got nil") + } + rc, ok := r.(io.ReadCloser) + if !ok { + rc = ioutil.NopCloser(r) + } + return &proxyReader{rc, b, time.Now()} +} + +// ID returs id of the bar. +func (b *Bar) ID() int { + select { + case b.operateState <- func(s *bState) { b.int64Ch <- int64(s.id) }: + return int(<-b.int64Ch) + case <-b.done: + return b.cacheState.id + } +} + +// Current returns bar's current number, in other words sum of all increments. +func (b *Bar) Current() int64 { + select { + case b.operateState <- func(s *bState) { b.int64Ch <- s.current }: + return <-b.int64Ch + case <-b.done: + return b.cacheState.current + } +} + +// SetTotal sets total dynamically. +// Set final to true, when total is known, it will trigger bar complete event. +func (b *Bar) SetTotal(total int64, final bool) bool { + select { + case b.operateState <- func(s *bState) { + if total > 0 { + s.total = total + } + if final { + s.current = s.total + s.toComplete = true + } + }: + return true + case <-b.done: + return false + } +} + +// SetRefill sets fill rune to r, up until n. +func (b *Bar) SetRefill(n int, r rune) { + if n <= 0 { + return + } + b.operateState <- func(s *bState) { + s.refill = &refill{r, int64(n)} + } +} + +// RefillBy is deprecated, use SetRefill +func (b *Bar) RefillBy(n int, r rune) { + b.SetRefill(n, r) +} + +// Increment is a shorthand for b.IncrBy(1). +func (b *Bar) Increment() { + b.IncrBy(1) +} + +// IncrBy increments progress bar by amount of n. +// wdd is optional work duration i.e. time.Since(start), +// which expected to be provided, if any ewma based decorator is used. +func (b *Bar) IncrBy(n int, wdd ...time.Duration) { + select { + case b.operateState <- func(s *bState) { + s.current += int64(n) + if s.current >= s.total { + s.current = s.total + s.toComplete = true + } + for _, ar := range s.amountReceivers { + ar.NextAmount(n, wdd...) + } + }: + case <-b.done: + } +} + +// Completed reports whether the bar is in completed state. +func (b *Bar) Completed() bool { + // omit select here, because primary usage of the method is for loop + // condition, like for !bar.Completed() {...} + // so when toComplete=true it is called once (at which time, the bar is still alive), + // then quits the loop and never suppose to be called afterwards. + return <-b.boolCh +} + +func (b *Bar) wSyncTable() [][]chan int { + select { + case b.operateState <- func(s *bState) { b.syncTableCh <- s.wSyncTable() }: + return <-b.syncTableCh + case <-b.done: + return b.cacheState.wSyncTable() + } +} + +func (b *Bar) serve(wg *sync.WaitGroup, s *bState, cancel <-chan struct{}) { + defer wg.Done() + for { + select { + case op := <-b.operateState: + op(s) + case b.boolCh <- s.toComplete: + case <-cancel: + s.toComplete = true + cancel = nil + case <-b.shutdown: + b.cacheState = s + close(b.done) + for _, sl := range s.shutdownListeners { + sl.Shutdown() + } + return + } + } +} + +func (b *Bar) render(debugOut io.Writer, tw int) { + select { + case b.operateState <- func(s *bState) { + defer func() { + // recovering if user defined decorator panics for example + if p := recover(); p != nil { + s.panicMsg = fmt.Sprintf("panic: %v", p) + fmt.Fprintf(debugOut, "%s %s bar id %02d %v\n", "[mpb]", time.Now(), s.id, s.panicMsg) + b.frameReaderCh <- &frameReader{ + Reader: strings.NewReader(fmt.Sprintf(fmt.Sprintf("%%.%ds\n", tw), s.panicMsg)), + toShutdown: true, + } + } + }() + r := s.draw(tw) + var extendedLines int + if s.newLineExtendFn != nil { + s.bufNL.Reset() + s.newLineExtendFn(s.bufNL, newStatistics(s)) + extendedLines = countLines(s.bufNL.Bytes()) + r = io.MultiReader(r, s.bufNL) + } + b.frameReaderCh <- &frameReader{ + Reader: r, + extendedLines: extendedLines, + toShutdown: s.toComplete && !s.completeFlushed, + removeOnComplete: s.removeOnComplete, + } + s.completeFlushed = s.toComplete + }: + case <-b.done: + s := b.cacheState + r := s.draw(tw) + var extendedLines int + if s.newLineExtendFn != nil { + s.bufNL.Reset() + s.newLineExtendFn(s.bufNL, newStatistics(s)) + extendedLines = countLines(s.bufNL.Bytes()) + r = io.MultiReader(r, s.bufNL) + } + b.frameReaderCh <- &frameReader{ + Reader: r, + extendedLines: extendedLines, + } + } +} + +func (s *bState) draw(termWidth int) io.Reader { + defer s.bufA.WriteByte('\n') + + if s.panicMsg != "" { + return strings.NewReader(fmt.Sprintf(fmt.Sprintf("%%.%ds\n", termWidth), s.panicMsg)) + } + + stat := newStatistics(s) + + for _, d := range s.pDecorators { + s.bufP.WriteString(d.Decor(stat)) + } + + for _, d := range s.aDecorators { + s.bufA.WriteString(d.Decor(stat)) + } + + prependCount := utf8.RuneCount(s.bufP.Bytes()) + appendCount := utf8.RuneCount(s.bufA.Bytes()) + + if s.barClearOnComplete && s.completeFlushed { + return io.MultiReader(s.bufP, s.bufA) + } + + s.fillBar(s.width) + barCount := utf8.RuneCount(s.bufB.Bytes()) + totalCount := prependCount + barCount + appendCount + if spaceCount := 0; totalCount > termWidth { + if !s.trimLeftSpace { + spaceCount++ + } + if !s.trimRightSpace { + spaceCount++ + } + s.fillBar(termWidth - prependCount - appendCount - spaceCount) + } + + return io.MultiReader(s.bufP, s.bufB, s.bufA) +} + +func (s *bState) fillBar(width int) { + defer func() { + s.bufB.WriteRune(s.runes[rRight]) + if !s.trimRightSpace { + s.bufB.WriteByte(' ') + } + }() + + s.bufB.Reset() + if !s.trimLeftSpace { + s.bufB.WriteByte(' ') + } + s.bufB.WriteRune(s.runes[rLeft]) + if width <= 2 { + return + } + + // bar s.width without leftEnd and rightEnd runes + barWidth := width - 2 + + completedWidth := internal.Percentage(s.total, s.current, int64(barWidth)) + + if s.refill != nil { + till := internal.Percentage(s.total, s.refill.till, int64(barWidth)) + // append refill rune + var i int64 + for i = 0; i < till; i++ { + s.bufB.WriteRune(s.refill.char) + } + for i = till; i < completedWidth; i++ { + s.bufB.WriteRune(s.runes[rFill]) + } + } else { + var i int64 + for i = 0; i < completedWidth; i++ { + s.bufB.WriteRune(s.runes[rFill]) + } + } + + if completedWidth < int64(barWidth) && completedWidth > 0 { + _, size := utf8.DecodeLastRune(s.bufB.Bytes()) + s.bufB.Truncate(s.bufB.Len() - size) + s.bufB.WriteRune(s.runes[rTip]) + } + + for i := completedWidth; i < int64(barWidth); i++ { + s.bufB.WriteRune(s.runes[rEmpty]) + } +} + +func (s *bState) wSyncTable() [][]chan int { + columns := make([]chan int, 0, len(s.pDecorators)+len(s.aDecorators)) + var pCount int + for _, d := range s.pDecorators { + if ok, ch := d.Syncable(); ok { + columns = append(columns, ch) + pCount++ + } + } + var aCount int + for _, d := range s.aDecorators { + if ok, ch := d.Syncable(); ok { + columns = append(columns, ch) + aCount++ + } + } + table := make([][]chan int, 2) + table[0] = columns[0:pCount] + table[1] = columns[pCount : pCount+aCount : pCount+aCount] + return table +} + +func newStatistics(s *bState) *decor.Statistics { + return &decor.Statistics{ + ID: s.id, + Completed: s.completeFlushed, + Total: s.total, + Current: s.current, + } +} + +func strToBarRunes(format string) (array barRunes) { + for i, n := 0, 0; len(format) > 0; i++ { + array[i], n = utf8.DecodeRuneInString(format) + format = format[n:] + } + return +} + +func countLines(b []byte) int { + return bytes.Count(b, []byte("\n")) +} diff --git a/vendor/github.com/vbauerster/mpb/bar_option.go b/vendor/github.com/vbauerster/mpb/bar_option.go new file mode 100644 index 000000000..e33bce4da --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/bar_option.go @@ -0,0 +1,124 @@ +package mpb + +import ( + "io" + + "github.com/vbauerster/mpb/decor" +) + +// BarOption is a function option which changes the default behavior of a bar, +// if passed to p.AddBar(int64, ...BarOption) +type BarOption func(*bState) + +// AppendDecorators let you inject decorators to the bar's right side +func AppendDecorators(appenders ...decor.Decorator) BarOption { + return func(s *bState) { + for _, decorator := range appenders { + if ar, ok := decorator.(decor.AmountReceiver); ok { + s.amountReceivers = append(s.amountReceivers, ar) + } + if sl, ok := decorator.(decor.ShutdownListener); ok { + s.shutdownListeners = append(s.shutdownListeners, sl) + } + s.aDecorators = append(s.aDecorators, decorator) + } + } +} + +// PrependDecorators let you inject decorators to the bar's left side +func PrependDecorators(prependers ...decor.Decorator) BarOption { + return func(s *bState) { + for _, decorator := range prependers { + if ar, ok := decorator.(decor.AmountReceiver); ok { + s.amountReceivers = append(s.amountReceivers, ar) + } + if sl, ok := decorator.(decor.ShutdownListener); ok { + s.shutdownListeners = append(s.shutdownListeners, sl) + } + s.pDecorators = append(s.pDecorators, decorator) + } + } +} + +// BarTrimLeft trims left side space of the bar +func BarTrimLeft() BarOption { + return func(s *bState) { + s.trimLeftSpace = true + } +} + +// BarTrimRight trims right space of the bar +func BarTrimRight() BarOption { + return func(s *bState) { + s.trimRightSpace = true + } +} + +// BarTrim trims both left and right spaces of the bar +func BarTrim() BarOption { + return func(s *bState) { + s.trimLeftSpace = true + s.trimRightSpace = true + } +} + +// BarID overwrites internal bar id +func BarID(id int) BarOption { + return func(s *bState) { + s.id = id + } +} + +// BarRemoveOnComplete is a flag, if set whole bar line will be removed on complete event. +// If both BarRemoveOnComplete and BarClearOnComplete are set, first bar section gets cleared +// and then whole bar line gets removed completely. +func BarRemoveOnComplete() BarOption { + return func(s *bState) { + s.removeOnComplete = true + } +} + +// BarReplaceOnComplete is indicator for delayed bar start, after the `runningBar` is complete. +// To achieve bar replacement effect, `runningBar` should has its `BarRemoveOnComplete` option set. +func BarReplaceOnComplete(runningBar *Bar) BarOption { + return func(s *bState) { + s.runningBar = runningBar + } +} + +// BarClearOnComplete is a flag, if set will clear bar section on complete event. +// If you need to remove a whole bar line, refer to BarRemoveOnComplete. +func BarClearOnComplete() BarOption { + return func(s *bState) { + s.barClearOnComplete = true + } +} + +// BarPriority sets bar's priority. +// Zero is highest priority, i.e. bar will be on top. +// If `BarReplaceOnComplete` option is supplied, this option is ignored. +func BarPriority(priority int) BarOption { + return func(s *bState) { + s.priority = priority + } +} + +// BarNewLineExtend takes user defined efn, which gets called each render cycle. +// Any write to provided writer of efn, will appear on new line of respective bar. +func BarNewLineExtend(efn func(io.Writer, *decor.Statistics)) BarOption { + return func(s *bState) { + s.newLineExtendFn = efn + } +} + +func barWidth(w int) BarOption { + return func(s *bState) { + s.width = w + } +} + +func barFormat(format string) BarOption { + return func(s *bState) { + s.runes = strToBarRunes(format) + } +} diff --git a/vendor/github.com/vbauerster/mpb/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/cwriter/writer.go new file mode 100644 index 000000000..0b1470d4c --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/cwriter/writer.go @@ -0,0 +1,78 @@ +package cwriter + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + + isatty "github.com/mattn/go-isatty" + "golang.org/x/crypto/ssh/terminal" +) + +// ESC is the ASCII code for escape character +const ESC = 27 + +var NotATTY = errors.New("not a terminal") + +var ( + cursorUp = fmt.Sprintf("%c[%dA", ESC, 1) + clearLine = fmt.Sprintf("%c[2K\r", ESC) + clearCursorAndLine = cursorUp + clearLine +) + +// Writer is a buffered the writer that updates the terminal. +// The contents of writer will be flushed when Flush is called. +type Writer struct { + out io.Writer + buf bytes.Buffer + isTerminal bool + fd int + lineCount int +} + +// New returns a new Writer with defaults +func New(out io.Writer) *Writer { + w := &Writer{out: out} + if f, ok := out.(*os.File); ok { + fd := f.Fd() + w.isTerminal = isatty.IsTerminal(fd) + w.fd = int(fd) + } + return w +} + +// Flush flushes the underlying buffer +func (w *Writer) Flush(lineCount int) error { + err := w.clearLines() + w.lineCount = lineCount + // WriteTo takes care of w.buf.Reset + if _, e := w.buf.WriteTo(w.out); err == nil { + err = e + } + return err +} + +// Write appends the contents of p to the underlying buffer +func (w *Writer) Write(p []byte) (n int, err error) { + return w.buf.Write(p) +} + +// WriteString writes string to the underlying buffer +func (w *Writer) WriteString(s string) (n int, err error) { + return w.buf.WriteString(s) +} + +// ReadFrom reads from the provided io.Reader and writes to the underlying buffer. +func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { + return w.buf.ReadFrom(r) +} + +func (w *Writer) GetWidth() (int, error) { + if w.isTerminal { + tw, _, err := terminal.GetSize(w.fd) + return tw, err + } + return -1, NotATTY +} diff --git a/vendor/github.com/vbauerster/mpb/cwriter/writer_posix.go b/vendor/github.com/vbauerster/mpb/cwriter/writer_posix.go new file mode 100644 index 000000000..05e31c480 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/cwriter/writer_posix.go @@ -0,0 +1,13 @@ +// +build !windows + +package cwriter + +import ( + "io" + "strings" +) + +func (w *Writer) clearLines() error { + _, err := io.WriteString(w.out, strings.Repeat(clearCursorAndLine, w.lineCount)) + return err +} diff --git a/vendor/github.com/vbauerster/mpb/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/cwriter/writer_windows.go new file mode 100644 index 000000000..dad7f50b2 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/cwriter/writer_windows.go @@ -0,0 +1,77 @@ +// +build windows + +package cwriter + +import ( + "io" + "strings" + "syscall" + "unsafe" + + "github.com/mattn/go-isatty" +) + +var kernel32 = syscall.NewLazyDLL("kernel32.dll") + +var ( + procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") + procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") + procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW") + procFillConsoleOutputAttribute = kernel32.NewProc("FillConsoleOutputAttribute") +) + +type ( + short int16 + word uint16 + dword uint32 + + coord struct { + x short + y short + } + smallRect struct { + left short + top short + right short + bottom short + } + consoleScreenBufferInfo struct { + size coord + cursorPosition coord + attributes word + window smallRect + maximumWindowSize coord + } +) + +// FdWriter is a writer with a file descriptor. +type FdWriter interface { + io.Writer + Fd() uintptr +} + +func (w *Writer) clearLines() error { + f, ok := w.out.(FdWriter) + if ok && !isatty.IsTerminal(f.Fd()) { + _, err := io.WriteString(w.out, strings.Repeat(clearCursorAndLine, w.lineCount)) + return err + } + fd := f.Fd() + var info consoleScreenBufferInfo + procGetConsoleScreenBufferInfo.Call(fd, uintptr(unsafe.Pointer(&info))) + + for i := 0; i < w.lineCount; i++ { + // move the cursor up + info.cursorPosition.y-- + procSetConsoleCursorPosition.Call(fd, uintptr(*(*int32)(unsafe.Pointer(&info.cursorPosition)))) + // clear the line + cursor := coord{ + x: info.window.left, + y: info.window.top + info.cursorPosition.y, + } + var count, w dword + count = dword(info.size.x) + procFillConsoleOutputCharacter.Call(fd, uintptr(' '), uintptr(count), *(*uintptr)(unsafe.Pointer(&cursor)), uintptr(unsafe.Pointer(&w))) + } + return nil +} diff --git a/vendor/github.com/vbauerster/mpb/decor/counters.go b/vendor/github.com/vbauerster/mpb/decor/counters.go new file mode 100644 index 000000000..e4161dc4b --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/counters.go @@ -0,0 +1,206 @@ +package decor + +import ( + "fmt" + "io" + "strconv" + "strings" +) + +const ( + _ = iota + KiB = 1 << (iota * 10) + MiB + GiB + TiB +) + +const ( + KB = 1000 + MB = KB * 1000 + GB = MB * 1000 + TB = GB * 1000 +) + +const ( + _ = iota + UnitKiB + UnitKB +) + +type CounterKiB int64 + +func (c CounterKiB) Format(st fmt.State, verb rune) { + prec, ok := st.Precision() + + if verb == 'd' || !ok { + prec = 0 + } + if verb == 'f' && !ok { + prec = 6 + } + // retain old beahavior if s verb used + if verb == 's' { + prec = 1 + } + + var res, unit string + switch { + case c >= TiB: + unit = "TiB" + res = strconv.FormatFloat(float64(c)/TiB, 'f', prec, 64) + case c >= GiB: + unit = "GiB" + res = strconv.FormatFloat(float64(c)/GiB, 'f', prec, 64) + case c >= MiB: + unit = "MiB" + res = strconv.FormatFloat(float64(c)/MiB, 'f', prec, 64) + case c >= KiB: + unit = "KiB" + res = strconv.FormatFloat(float64(c)/KiB, 'f', prec, 64) + default: + unit = "b" + res = strconv.FormatInt(int64(c), 10) + } + + if st.Flag(' ') { + res += " " + } + res += unit + + if w, ok := st.Width(); ok { + if len(res) < w { + pad := strings.Repeat(" ", w-len(res)) + if st.Flag(int('-')) { + res += pad + } else { + res = pad + res + } + } + } + + io.WriteString(st, res) +} + +type CounterKB int64 + +func (c CounterKB) Format(st fmt.State, verb rune) { + prec, ok := st.Precision() + + if verb == 'd' || !ok { + prec = 0 + } + if verb == 'f' && !ok { + prec = 6 + } + // retain old beahavior if s verb used + if verb == 's' { + prec = 1 + } + + var res, unit string + switch { + case c >= TB: + unit = "TB" + res = strconv.FormatFloat(float64(c)/TB, 'f', prec, 64) + case c >= GB: + unit = "GB" + res = strconv.FormatFloat(float64(c)/GB, 'f', prec, 64) + case c >= MB: + unit = "MB" + res = strconv.FormatFloat(float64(c)/MB, 'f', prec, 64) + case c >= KB: + unit = "kB" + res = strconv.FormatFloat(float64(c)/KB, 'f', prec, 64) + default: + unit = "b" + res = strconv.FormatInt(int64(c), 10) + } + + if st.Flag(' ') { + res += " " + } + res += unit + + if w, ok := st.Width(); ok { + if len(res) < w { + pad := strings.Repeat(" ", w-len(res)) + if st.Flag(int('-')) { + res += pad + } else { + res = pad + res + } + } + } + + io.WriteString(st, res) +} + +// CountersNoUnit is a wrapper around Counters with no unit param. +func CountersNoUnit(pairFormat string, wcc ...WC) Decorator { + return Counters(0, pairFormat, wcc...) +} + +// CountersKibiByte is a wrapper around Counters with predefined unit UnitKiB (bytes/1024). +func CountersKibiByte(pairFormat string, wcc ...WC) Decorator { + return Counters(UnitKiB, pairFormat, wcc...) +} + +// CountersKiloByte is a wrapper around Counters with predefined unit UnitKB (bytes/1000). +func CountersKiloByte(pairFormat string, wcc ...WC) Decorator { + return Counters(UnitKB, pairFormat, wcc...) +} + +// Counters decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `pairFormat` printf compatible verbs for current and total, like "%f" or "%d" +// +// `wcc` optional WC config +// +// pairFormat example if UnitKB is chosen: +// +// "%.1f / %.1f" = "1.0MB / 12.0MB" or "% .1f / % .1f" = "1.0 MB / 12.0 MB" +func Counters(unit int, pairFormat string, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &countersDecorator{ + WC: wc, + unit: unit, + pairFormat: pairFormat, + } + return d +} + +type countersDecorator struct { + WC + unit int + pairFormat string + completeMsg *string +} + +func (d *countersDecorator) Decor(st *Statistics) string { + if st.Completed && d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + + var str string + switch d.unit { + case UnitKiB: + str = fmt.Sprintf(d.pairFormat, CounterKiB(st.Current), CounterKiB(st.Total)) + case UnitKB: + str = fmt.Sprintf(d.pairFormat, CounterKB(st.Current), CounterKB(st.Total)) + default: + str = fmt.Sprintf(d.pairFormat, st.Current, st.Total) + } + + return d.FormatMsg(str) +} + +func (d *countersDecorator) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} diff --git a/vendor/github.com/vbauerster/mpb/decor/decorator.go b/vendor/github.com/vbauerster/mpb/decor/decorator.go new file mode 100644 index 000000000..6aaf6c830 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/decorator.go @@ -0,0 +1,144 @@ +package decor + +import ( + "fmt" + "time" + "unicode/utf8" +) + +const ( + // DidentRight bit specifies identation direction. + // |foo |b | With DidentRight + // | foo| b| Without DidentRight + DidentRight = 1 << iota + + // DextraSpace bit adds extra space, makes sense with DSyncWidth only. + // When DidentRight bit set, the space will be added to the right, + // otherwise to the left. + DextraSpace + + // DSyncWidth bit enables same column width synchronization. + // Effective with multiple bars only. + DSyncWidth + + // DSyncWidthR is shortcut for DSyncWidth|DidentRight + DSyncWidthR = DSyncWidth | DidentRight + + // DSyncSpace is shortcut for DSyncWidth|DextraSpace + DSyncSpace = DSyncWidth | DextraSpace + + // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight + DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight +) + +const ( + ET_STYLE_GO = iota + ET_STYLE_HHMMSS + ET_STYLE_HHMM + ET_STYLE_MMSS +) + +// Statistics is a struct, which gets passed to a Decorator. +type Statistics struct { + ID int + Completed bool + Total int64 + Current int64 +} + +// Decorator interface. +// A decorator must implement this interface, in order to be used with mpb library. +type Decorator interface { + Decor(*Statistics) string + Syncable +} + +// Syncable interface. +// All decorators implement this interface implicitly. +// Its Syncable method exposes width sync channel, if sync is enabled. +type Syncable interface { + Syncable() (bool, chan int) +} + +// OnCompleteMessenger interface. +// Decorators implementing this interface suppose to return provided string on complete event. +type OnCompleteMessenger interface { + OnCompleteMessage(string) +} + +// AmountReceiver interface. +// If decorator needs to receive increment amount, +// so this is the right interface to implement. +type AmountReceiver interface { + NextAmount(int, ...time.Duration) +} + +// ShutdownListener interface. +// If decorator needs to be notified once upon bar shutdown event, +// so this is the right interface to implement. +type ShutdownListener interface { + Shutdown() +} + +// Global convenience shortcuts +var ( + WCSyncWidth = WC{C: DSyncWidth} + WCSyncWidthR = WC{C: DSyncWidthR} + WCSyncSpace = WC{C: DSyncSpace} + WCSyncSpaceR = WC{C: DSyncSpaceR} +) + +// WC is a struct with two public fields W and C, both of int type. +// W represents width and C represents bit set of width related config. +type WC struct { + W int + C int + format string + wsync chan int +} + +// FormatMsg formats final message according to WC.W and WC.C. +// Should be called by any Decorator implementation. +func (wc WC) FormatMsg(msg string) string { + if (wc.C & DSyncWidth) != 0 { + wc.wsync <- utf8.RuneCountInString(msg) + max := <-wc.wsync + if max == 0 { + max = wc.W + } + if (wc.C & DextraSpace) != 0 { + max++ + } + return fmt.Sprintf(fmt.Sprintf(wc.format, max), msg) + } + return fmt.Sprintf(fmt.Sprintf(wc.format, wc.W), msg) +} + +// Init initializes width related config. +func (wc *WC) Init() { + wc.format = "%%" + if (wc.C & DidentRight) != 0 { + wc.format += "-" + } + wc.format += "%ds" + if (wc.C & DSyncWidth) != 0 { + wc.wsync = make(chan int) + } +} + +func (wc *WC) Syncable() (bool, chan int) { + return (wc.C & DSyncWidth) != 0, wc.wsync +} + +// OnComplete returns decorator, which wraps provided decorator, with sole +// purpose to display provided message on complete event. +// +// `decorator` Decorator to wrap +// +// `message` message to display on complete event +func OnComplete(decorator Decorator, message string) Decorator { + if d, ok := decorator.(OnCompleteMessenger); ok { + d.OnCompleteMessage(message) + } + return decorator +} diff --git a/vendor/github.com/vbauerster/mpb/decor/doc.go b/vendor/github.com/vbauerster/mpb/decor/doc.go new file mode 100644 index 000000000..561a8677c --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/doc.go @@ -0,0 +1,25 @@ +// Copyright (C) 2016-2018 Vladimir Bauer +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package decor contains common decorators used by "github.com/vbauerster/mpb" package. + + Some decorators returned by this package might have a closure state. It is ok to use + decorators concurrently, unless you share the same decorator among multiple + *mpb.Bar instances. To avoid data races, create new decorator per *mpb.Bar instance. + + Don't: + + p := mpb.New() + name := decor.Name("bar") + p.AddBar(100, mpb.AppendDecorators(name)) + p.AddBar(100, mpb.AppendDecorators(name)) + + Do: + + p := mpb.New() + p.AddBar(100, mpb.AppendDecorators(decor.Name("bar1"))) + p.AddBar(100, mpb.AppendDecorators(decor.Name("bar2"))) +*/ +package decor diff --git a/vendor/github.com/vbauerster/mpb/decor/elapsed.go b/vendor/github.com/vbauerster/mpb/decor/elapsed.go new file mode 100644 index 000000000..649d40a30 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/elapsed.go @@ -0,0 +1,68 @@ +package decor + +import ( + "fmt" + "time" +) + +// Elapsed returns elapsed time decorator. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `wcc` optional WC config +func Elapsed(style int, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &elapsedDecorator{ + WC: wc, + style: style, + startTime: time.Now(), + } + return d +} + +type elapsedDecorator struct { + WC + style int + startTime time.Time + msg string + completeMsg *string +} + +func (d *elapsedDecorator) Decor(st *Statistics) string { + if st.Completed { + if d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + return d.FormatMsg(d.msg) + } + + timeElapsed := time.Since(d.startTime) + hours := int64((timeElapsed / time.Hour) % 60) + minutes := int64((timeElapsed / time.Minute) % 60) + seconds := int64((timeElapsed / time.Second) % 60) + + switch d.style { + case ET_STYLE_GO: + d.msg = fmt.Sprint(time.Duration(timeElapsed.Seconds()) * time.Second) + case ET_STYLE_HHMMSS: + d.msg = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + case ET_STYLE_HHMM: + d.msg = fmt.Sprintf("%02d:%02d", hours, minutes) + case ET_STYLE_MMSS: + if hours > 0 { + d.msg = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + } else { + d.msg = fmt.Sprintf("%02d:%02d", minutes, seconds) + } + } + + return d.FormatMsg(d.msg) +} + +func (d *elapsedDecorator) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} diff --git a/vendor/github.com/vbauerster/mpb/decor/eta.go b/vendor/github.com/vbauerster/mpb/decor/eta.go new file mode 100644 index 000000000..44a1f03ea --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/eta.go @@ -0,0 +1,207 @@ +package decor + +import ( + "fmt" + "math" + "time" + + "github.com/VividCortex/ewma" + "github.com/vbauerster/mpb/internal" +) + +type TimeNormalizer func(time.Duration) time.Duration + +// EwmaETA exponential-weighted-moving-average based ETA decorator. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `age` is the previous N samples to average over. +// +// `wcc` optional WC config +func EwmaETA(style int, age float64, wcc ...WC) Decorator { + return MovingAverageETA(style, ewma.NewMovingAverage(age), NopNormalizer(), wcc...) +} + +// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `average` available implementations of MovingAverage [ewma.MovingAverage|NewMedian|NewMedianEwma] +// +// `normalizer` available implementations are [NopNormalizer|FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer] +// +// `wcc` optional WC config +func MovingAverageETA(style int, average MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &movingAverageETA{ + WC: wc, + style: style, + average: average, + normalizer: normalizer, + } + return d +} + +type movingAverageETA struct { + WC + style int + average ewma.MovingAverage + completeMsg *string + normalizer TimeNormalizer +} + +func (d *movingAverageETA) Decor(st *Statistics) string { + if st.Completed && d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + + v := internal.Round(d.average.Value()) + remaining := d.normalizer(time.Duration((st.Total - st.Current) * int64(v))) + hours := int64((remaining / time.Hour) % 60) + minutes := int64((remaining / time.Minute) % 60) + seconds := int64((remaining / time.Second) % 60) + + var str string + switch d.style { + case ET_STYLE_GO: + str = fmt.Sprint(time.Duration(remaining.Seconds()) * time.Second) + case ET_STYLE_HHMMSS: + str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + case ET_STYLE_HHMM: + str = fmt.Sprintf("%02d:%02d", hours, minutes) + case ET_STYLE_MMSS: + if hours > 0 { + str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + } else { + str = fmt.Sprintf("%02d:%02d", minutes, seconds) + } + } + + return d.FormatMsg(str) +} + +func (d *movingAverageETA) NextAmount(n int, wdd ...time.Duration) { + var workDuration time.Duration + for _, wd := range wdd { + workDuration = wd + } + lastItemEstimate := float64(workDuration) / float64(n) + if math.IsInf(lastItemEstimate, 0) || math.IsNaN(lastItemEstimate) { + return + } + d.average.Add(lastItemEstimate) +} + +func (d *movingAverageETA) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} + +// AverageETA decorator. +// +// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] +// +// `wcc` optional WC config +func AverageETA(style int, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &averageETA{ + WC: wc, + style: style, + startTime: time.Now(), + } + return d +} + +type averageETA struct { + WC + style int + startTime time.Time + completeMsg *string +} + +func (d *averageETA) Decor(st *Statistics) string { + if st.Completed && d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + + var str string + timeElapsed := time.Since(d.startTime) + v := internal.Round(float64(timeElapsed) / float64(st.Current)) + if math.IsInf(v, 0) || math.IsNaN(v) { + v = 0 + } + remaining := time.Duration((st.Total - st.Current) * int64(v)) + hours := int64((remaining / time.Hour) % 60) + minutes := int64((remaining / time.Minute) % 60) + seconds := int64((remaining / time.Second) % 60) + + switch d.style { + case ET_STYLE_GO: + str = fmt.Sprint(time.Duration(remaining.Seconds()) * time.Second) + case ET_STYLE_HHMMSS: + str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + case ET_STYLE_HHMM: + str = fmt.Sprintf("%02d:%02d", hours, minutes) + case ET_STYLE_MMSS: + if hours > 0 { + str = fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + } else { + str = fmt.Sprintf("%02d:%02d", minutes, seconds) + } + } + + return d.FormatMsg(str) +} + +func (d *averageETA) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} + +func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer { + var normalized time.Duration + var lastCall time.Time + return func(remaining time.Duration) time.Duration { + if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < maxTolerate/2 { + normalized = remaining + lastCall = time.Now() + return remaining + } + normalized -= time.Since(lastCall) + lastCall = time.Now() + return normalized + } +} + +func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer { + var normalized time.Duration + var lastCall time.Time + var count int + return func(remaining time.Duration) time.Duration { + if count == 0 || remaining <= time.Duration(15*time.Second) { + count = updInterval + normalized = remaining + lastCall = time.Now() + return remaining + } + count-- + normalized -= time.Since(lastCall) + lastCall = time.Now() + if normalized > 0 { + return normalized + } + return remaining + } +} + +func NopNormalizer() TimeNormalizer { + return func(remaining time.Duration) time.Duration { + return remaining + } +} diff --git a/vendor/github.com/vbauerster/mpb/decor/moving-average.go b/vendor/github.com/vbauerster/mpb/decor/moving-average.go new file mode 100644 index 000000000..f9596a27f --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/moving-average.go @@ -0,0 +1,66 @@ +package decor + +import ( + "sort" + + "github.com/VividCortex/ewma" +) + +// MovingAverage is the interface that computes a moving average over a time- +// series stream of numbers. The average may be over a window or exponentially +// decaying. +type MovingAverage interface { + Add(float64) + Value() float64 + Set(float64) +} + +type medianWindow [3]float64 + +func (s *medianWindow) Len() int { return len(s) } +func (s *medianWindow) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s *medianWindow) Less(i, j int) bool { return s[i] < s[j] } + +func (s *medianWindow) Add(value float64) { + s[0], s[1] = s[1], s[2] + s[2] = value +} + +func (s *medianWindow) Value() float64 { + tmp := *s + sort.Sort(&tmp) + return tmp[1] +} + +func (s *medianWindow) Set(value float64) { + for i := 0; i < len(s); i++ { + s[i] = value + } +} + +// NewMedian is fixed last 3 samples median MovingAverage. +func NewMedian() MovingAverage { + return new(medianWindow) +} + +type medianEwma struct { + count uint + median MovingAverage + MovingAverage +} + +func (s *medianEwma) Add(v float64) { + s.median.Add(v) + if s.count >= 2 { + s.MovingAverage.Add(s.median.Value()) + } + s.count++ +} + +// NewMedianEwma is ewma based MovingAverage, which gets its values from median MovingAverage. +func NewMedianEwma(age ...float64) MovingAverage { + return &medianEwma{ + MovingAverage: ewma.NewMovingAverage(age...), + median: NewMedian(), + } +} diff --git a/vendor/github.com/vbauerster/mpb/decor/name.go b/vendor/github.com/vbauerster/mpb/decor/name.go new file mode 100644 index 000000000..a5a5d1469 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/name.go @@ -0,0 +1,45 @@ +package decor + +// StaticName returns name decorator. +// +// `name` string to display +// +// `wcc` optional WC config +func StaticName(name string, wcc ...WC) Decorator { + return Name(name, wcc...) +} + +// Name returns name decorator. +// +// `name` string to display +// +// `wcc` optional WC config +func Name(name string, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &nameDecorator{ + WC: wc, + msg: name, + } + return d +} + +type nameDecorator struct { + WC + msg string + complete *string +} + +func (d *nameDecorator) Decor(st *Statistics) string { + if st.Completed && d.complete != nil { + return d.FormatMsg(*d.complete) + } + return d.FormatMsg(d.msg) +} + +func (d *nameDecorator) OnCompleteMessage(msg string) { + d.complete = &msg +} diff --git a/vendor/github.com/vbauerster/mpb/decor/percentage.go b/vendor/github.com/vbauerster/mpb/decor/percentage.go new file mode 100644 index 000000000..078fbcf89 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/percentage.go @@ -0,0 +1,39 @@ +package decor + +import ( + "fmt" + + "github.com/vbauerster/mpb/internal" +) + +// Percentage returns percentage decorator. +// +// `wcc` optional WC config +func Percentage(wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &percentageDecorator{ + WC: wc, + } + return d +} + +type percentageDecorator struct { + WC + completeMsg *string +} + +func (d *percentageDecorator) Decor(st *Statistics) string { + if st.Completed && d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + str := fmt.Sprintf("%d %%", internal.Percentage(st.Total, st.Current, 100)) + return d.FormatMsg(str) +} + +func (d *percentageDecorator) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} diff --git a/vendor/github.com/vbauerster/mpb/decor/speed.go b/vendor/github.com/vbauerster/mpb/decor/speed.go new file mode 100644 index 000000000..395e5d04d --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/decor/speed.go @@ -0,0 +1,270 @@ +package decor + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + "github.com/VividCortex/ewma" +) + +type SpeedKiB float64 + +func (s SpeedKiB) Format(st fmt.State, verb rune) { + prec, ok := st.Precision() + + if verb == 'd' || !ok { + prec = 0 + } + if verb == 'f' && !ok { + prec = 6 + } + // retain old beahavior if s verb used + if verb == 's' { + prec = 1 + } + + var res, unit string + switch { + case s >= TiB: + unit = "TiB/s" + res = strconv.FormatFloat(float64(s)/TiB, 'f', prec, 64) + case s >= GiB: + unit = "GiB/s" + res = strconv.FormatFloat(float64(s)/GiB, 'f', prec, 64) + case s >= MiB: + unit = "MiB/s" + res = strconv.FormatFloat(float64(s)/MiB, 'f', prec, 64) + case s >= KiB: + unit = "KiB/s" + res = strconv.FormatFloat(float64(s)/KiB, 'f', prec, 64) + default: + unit = "b/s" + res = strconv.FormatInt(int64(s), 10) + } + + if st.Flag(' ') { + res += " " + } + res += unit + + if w, ok := st.Width(); ok { + if len(res) < w { + pad := strings.Repeat(" ", w-len(res)) + if st.Flag(int('-')) { + res += pad + } else { + res = pad + res + } + } + } + + io.WriteString(st, res) +} + +type SpeedKB float64 + +func (s SpeedKB) Format(st fmt.State, verb rune) { + prec, ok := st.Precision() + + if verb == 'd' || !ok { + prec = 0 + } + if verb == 'f' && !ok { + prec = 6 + } + // retain old beahavior if s verb used + if verb == 's' { + prec = 1 + } + + var res, unit string + switch { + case s >= TB: + unit = "TB/s" + res = strconv.FormatFloat(float64(s)/TB, 'f', prec, 64) + case s >= GB: + unit = "GB/s" + res = strconv.FormatFloat(float64(s)/GB, 'f', prec, 64) + case s >= MB: + unit = "MB/s" + res = strconv.FormatFloat(float64(s)/MB, 'f', prec, 64) + case s >= KB: + unit = "kB/s" + res = strconv.FormatFloat(float64(s)/KB, 'f', prec, 64) + default: + unit = "b/s" + res = strconv.FormatInt(int64(s), 10) + } + + if st.Flag(' ') { + res += " " + } + res += unit + + if w, ok := st.Width(); ok { + if len(res) < w { + pad := strings.Repeat(" ", w-len(res)) + if st.Flag(int('-')) { + res += pad + } else { + res = pad + res + } + } + } + + io.WriteString(st, res) +} + +// EwmaSpeed exponential-weighted-moving-average based speed decorator, +// with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `unitFormat` printf compatible verb for value, like "%f" or "%d" +// +// `average` MovingAverage implementation +// +// `wcc` optional WC config +// +// unitFormat example if UnitKiB is chosen: +// +// "%.1f" = "1.0MiB/s" or "% .1f" = "1.0 MiB/s" +func EwmaSpeed(unit int, unitFormat string, age float64, wcc ...WC) Decorator { + return MovingAverageSpeed(unit, unitFormat, ewma.NewMovingAverage(age), wcc...) +} + +// MovingAverageSpeed decorator relies on MovingAverage implementation to calculate its average. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `unitFormat` printf compatible verb for value, like "%f" or "%d" +// +// `average` MovingAverage implementation +// +// `wcc` optional WC config +func MovingAverageSpeed(unit int, unitFormat string, average MovingAverage, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &movingAverageSpeed{ + WC: wc, + unit: unit, + unitFormat: unitFormat, + average: average, + } + return d +} + +type movingAverageSpeed struct { + WC + unit int + unitFormat string + average ewma.MovingAverage + msg string + completeMsg *string +} + +func (d *movingAverageSpeed) Decor(st *Statistics) string { + if st.Completed { + if d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + return d.FormatMsg(d.msg) + } + + speed := d.average.Value() + switch d.unit { + case UnitKiB: + d.msg = fmt.Sprintf(d.unitFormat, SpeedKiB(speed)) + case UnitKB: + d.msg = fmt.Sprintf(d.unitFormat, SpeedKB(speed)) + default: + d.msg = fmt.Sprintf(d.unitFormat, speed) + } + + return d.FormatMsg(d.msg) +} + +func (s *movingAverageSpeed) NextAmount(n int, wdd ...time.Duration) { + var workDuration time.Duration + for _, wd := range wdd { + workDuration = wd + } + speed := float64(n) / workDuration.Seconds() / 1000 + if math.IsInf(speed, 0) || math.IsNaN(speed) { + return + } + s.average.Add(speed) +} + +func (d *movingAverageSpeed) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} + +// AverageSpeed decorator with dynamic unit measure adjustment. +// +// `unit` one of [0|UnitKiB|UnitKB] zero for no unit +// +// `unitFormat` printf compatible verb for value, like "%f" or "%d" +// +// `wcc` optional WC config +// +// unitFormat example if UnitKiB is chosen: +// +// "%.1f" = "1.0MiB/s" or "% .1f" = "1.0 MiB/s" +func AverageSpeed(unit int, unitFormat string, wcc ...WC) Decorator { + var wc WC + for _, widthConf := range wcc { + wc = widthConf + } + wc.Init() + d := &averageSpeed{ + WC: wc, + unit: unit, + unitFormat: unitFormat, + startTime: time.Now(), + } + return d +} + +type averageSpeed struct { + WC + unit int + unitFormat string + startTime time.Time + msg string + completeMsg *string +} + +func (d *averageSpeed) Decor(st *Statistics) string { + if st.Completed { + if d.completeMsg != nil { + return d.FormatMsg(*d.completeMsg) + } + return d.FormatMsg(d.msg) + } + + timeElapsed := time.Since(d.startTime) + speed := float64(st.Current) / timeElapsed.Seconds() + + switch d.unit { + case UnitKiB: + d.msg = fmt.Sprintf(d.unitFormat, SpeedKiB(speed)) + case UnitKB: + d.msg = fmt.Sprintf(d.unitFormat, SpeedKB(speed)) + default: + d.msg = fmt.Sprintf(d.unitFormat, speed) + } + + return d.FormatMsg(d.msg) +} + +func (d *averageSpeed) OnCompleteMessage(msg string) { + d.completeMsg = &msg +} diff --git a/vendor/github.com/vbauerster/mpb/doc.go b/vendor/github.com/vbauerster/mpb/doc.go new file mode 100644 index 000000000..16245956a --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/doc.go @@ -0,0 +1,6 @@ +// Copyright (C) 2016-2018 Vladimir Bauer +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package mpb is a library for rendering progress bars in terminal applications. +package mpb diff --git a/vendor/github.com/vbauerster/mpb/internal/percentage.go b/vendor/github.com/vbauerster/mpb/internal/percentage.go new file mode 100644 index 000000000..3c8defb7d --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/internal/percentage.go @@ -0,0 +1,10 @@ +package internal + +// Percentage is a helper function, to calculate percentage. +func Percentage(total, current, width int64) int64 { + if total <= 0 { + return 0 + } + p := float64(width*current) / float64(total) + return int64(Round(p)) +} diff --git a/vendor/github.com/vbauerster/mpb/internal/round.go b/vendor/github.com/vbauerster/mpb/internal/round.go new file mode 100644 index 000000000..c54a789d2 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/internal/round.go @@ -0,0 +1,49 @@ +package internal + +import "math" + +const ( + uvone = 0x3FF0000000000000 + mask = 0x7FF + shift = 64 - 11 - 1 + bias = 1023 + signMask = 1 << 63 + fracMask = 1<<shift - 1 +) + +// Round returns the nearest integer, rounding half away from zero. +// +// Special cases are: +// Round(±0) = ±0 +// Round(±Inf) = ±Inf +// Round(NaN) = NaN +func Round(x float64) float64 { + // Round is a faster implementation of: + // + // func Round(x float64) float64 { + // t := Trunc(x) + // if Abs(x-t) >= 0.5 { + // return t + Copysign(1, x) + // } + // return t + // } + bits := math.Float64bits(x) + e := uint(bits>>shift) & mask + if e < bias { + // Round abs(x) < 1 including denormals. + bits &= signMask // +-0 + if e == bias-1 { + bits |= uvone // +-1 + } + } else if e < bias+shift { + // Round any abs(x) >= 1 containing a fractional component [0,1). + // + // Numbers with larger exponents are returned unchanged since they + // must be either an integer, infinity, or NaN. + const half = 1 << (shift - 1) + e -= bias + bits += half >> e + bits &^= fracMask >> e + } + return math.Float64frombits(bits) +} diff --git a/vendor/github.com/vbauerster/mpb/options.go b/vendor/github.com/vbauerster/mpb/options.go new file mode 100644 index 000000000..05d2ecf1f --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/options.go @@ -0,0 +1,95 @@ +package mpb + +import ( + "io" + "sync" + "time" + "unicode/utf8" + + "github.com/vbauerster/mpb/cwriter" +) + +// ProgressOption is a function option which changes the default behavior of +// progress pool, if passed to mpb.New(...ProgressOption) +type ProgressOption func(*pState) + +// WithWaitGroup provides means to have a single joint point. +// If *sync.WaitGroup is provided, you can safely call just p.Wait() +// without calling Wait() on provided *sync.WaitGroup. +// Makes sense when there are more than one bar to render. +func WithWaitGroup(wg *sync.WaitGroup) ProgressOption { + return func(s *pState) { + s.uwg = wg + } +} + +// WithWidth overrides default width 80 +func WithWidth(w int) ProgressOption { + return func(s *pState) { + if w >= 0 { + s.width = w + } + } +} + +// WithFormat overrides default bar format "[=>-]" +func WithFormat(format string) ProgressOption { + return func(s *pState) { + if utf8.RuneCountInString(format) == formatLen { + s.format = format + } + } +} + +// WithRefreshRate overrides default 120ms refresh rate +func WithRefreshRate(d time.Duration) ProgressOption { + return func(s *pState) { + if d < 10*time.Millisecond { + return + } + s.rr = d + } +} + +// WithManualRefresh disables internal auto refresh time.Ticker. +// Refresh will occur upon receive value from provided ch. +func WithManualRefresh(ch <-chan time.Time) ProgressOption { + return func(s *pState) { + s.manualRefreshCh = ch + } +} + +// WithCancel provide your cancel channel, +// which you plan to close at some point. +func WithCancel(ch <-chan struct{}) ProgressOption { + return func(s *pState) { + s.cancel = ch + } +} + +// WithShutdownNotifier provided chanel will be closed, after all bars have been rendered. +func WithShutdownNotifier(ch chan struct{}) ProgressOption { + return func(s *pState) { + s.shutdownNotifier = ch + } +} + +// WithOutput overrides default output os.Stdout +func WithOutput(w io.Writer) ProgressOption { + return func(s *pState) { + if w == nil { + return + } + s.cw = cwriter.New(w) + } +} + +// WithDebugOutput sets debug output. +func WithDebugOutput(w io.Writer) ProgressOption { + return func(s *pState) { + if w == nil { + return + } + s.debugOut = w + } +} diff --git a/vendor/github.com/vbauerster/mpb/options_go1.7.go b/vendor/github.com/vbauerster/mpb/options_go1.7.go new file mode 100644 index 000000000..ca9a5bad8 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/options_go1.7.go @@ -0,0 +1,15 @@ +//+build go1.7 + +package mpb + +import "context" + +// WithContext provided context will be used for cancellation purposes +func WithContext(ctx context.Context) ProgressOption { + return func(s *pState) { + if ctx == nil { + panic("ctx must not be nil") + } + s.cancel = ctx.Done() + } +} diff --git a/vendor/github.com/vbauerster/mpb/priority_queue.go b/vendor/github.com/vbauerster/mpb/priority_queue.go new file mode 100644 index 000000000..7bc588c29 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/priority_queue.go @@ -0,0 +1,40 @@ +package mpb + +import "container/heap" + +// A priorityQueue implements heap.Interface +type priorityQueue []*Bar + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].priority < pq[j].priority +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + bar := x.(*Bar) + bar.index = n + *pq = append(*pq, bar) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + bar := old[n-1] + bar.index = -1 // for safety + *pq = old[0 : n-1] + return bar +} + +// update modifies the priority of a Bar in the queue. +func (pq *priorityQueue) update(bar *Bar, priority int) { + bar.priority = priority + heap.Fix(pq, bar.index) +} diff --git a/vendor/github.com/vbauerster/mpb/progress.go b/vendor/github.com/vbauerster/mpb/progress.go new file mode 100644 index 000000000..d95fe45b7 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/progress.go @@ -0,0 +1,251 @@ +package mpb + +import ( + "container/heap" + "fmt" + "io" + "io/ioutil" + "os" + "sync" + "time" + + "github.com/vbauerster/mpb/cwriter" +) + +const ( + // default RefreshRate + prr = 120 * time.Millisecond + // default width + pwidth = 80 + // default format + pformat = "[=>-]" +) + +// Progress represents the container that renders Progress bars +type Progress struct { + wg *sync.WaitGroup + uwg *sync.WaitGroup + operateState chan func(*pState) + done chan struct{} +} + +type pState struct { + bHeap *priorityQueue + shutdownPending []*Bar + heapUpdated bool + zeroWait bool + idCounter int + width int + format string + rr time.Duration + cw *cwriter.Writer + pMatrix map[int][]chan int + aMatrix map[int][]chan int + + // following are provided by user + uwg *sync.WaitGroup + manualRefreshCh <-chan time.Time + cancel <-chan struct{} + shutdownNotifier chan struct{} + waitBars map[*Bar]*Bar + debugOut io.Writer +} + +// New creates new Progress instance, which orchestrates bars rendering process. +// Accepts mpb.ProgressOption funcs for customization. +func New(options ...ProgressOption) *Progress { + pq := make(priorityQueue, 0) + heap.Init(&pq) + s := &pState{ + bHeap: &pq, + width: pwidth, + format: pformat, + cw: cwriter.New(os.Stdout), + rr: prr, + waitBars: make(map[*Bar]*Bar), + debugOut: ioutil.Discard, + } + + for _, opt := range options { + if opt != nil { + opt(s) + } + } + + p := &Progress{ + uwg: s.uwg, + wg: new(sync.WaitGroup), + operateState: make(chan func(*pState)), + done: make(chan struct{}), + } + go p.serve(s) + return p +} + +// AddBar creates a new progress bar and adds to the container. +func (p *Progress) AddBar(total int64, options ...BarOption) *Bar { + p.wg.Add(1) + result := make(chan *Bar) + select { + case p.operateState <- func(s *pState) { + options = append(options, barWidth(s.width), barFormat(s.format)) + b := newBar(p.wg, s.idCounter, total, s.cancel, options...) + if b.runningBar != nil { + s.waitBars[b.runningBar] = b + } else { + heap.Push(s.bHeap, b) + s.heapUpdated = true + } + s.idCounter++ + result <- b + }: + return <-result + case <-p.done: + p.wg.Done() + return nil + } +} + +// Abort is only effective while bar progress is running, +// it means remove bar now without waiting for its completion. +// If bar is already completed, there is nothing to abort. +// If you need to remove bar after completion, use BarRemoveOnComplete BarOption. +func (p *Progress) Abort(b *Bar, remove bool) { + select { + case p.operateState <- func(s *pState) { + if b.index < 0 { + return + } + if remove { + s.heapUpdated = heap.Remove(s.bHeap, b.index) != nil + } + s.shutdownPending = append(s.shutdownPending, b) + }: + case <-p.done: + } +} + +// UpdateBarPriority provides a way to change bar's order position. +// Zero is highest priority, i.e. bar will be on top. +func (p *Progress) UpdateBarPriority(b *Bar, priority int) { + select { + case p.operateState <- func(s *pState) { s.bHeap.update(b, priority) }: + case <-p.done: + } +} + +// BarCount returns bars count +func (p *Progress) BarCount() int { + result := make(chan int, 1) + select { + case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }: + return <-result + case <-p.done: + return 0 + } +} + +// Wait first waits for user provided *sync.WaitGroup, if any, +// then waits far all bars to complete and finally shutdowns master goroutine. +// After this method has been called, there is no way to reuse *Progress instance. +func (p *Progress) Wait() { + if p.uwg != nil { + p.uwg.Wait() + } + + p.wg.Wait() + + select { + case p.operateState <- func(s *pState) { s.zeroWait = true }: + <-p.done + case <-p.done: + } +} + +func (s *pState) updateSyncMatrix() { + s.pMatrix = make(map[int][]chan int) + s.aMatrix = make(map[int][]chan int) + for i := 0; i < s.bHeap.Len(); i++ { + bar := (*s.bHeap)[i] + table := bar.wSyncTable() + pRow, aRow := table[0], table[1] + + for i, ch := range pRow { + s.pMatrix[i] = append(s.pMatrix[i], ch) + } + + for i, ch := range aRow { + s.aMatrix[i] = append(s.aMatrix[i], ch) + } + } +} + +func (s *pState) render(tw int) { + if s.heapUpdated { + s.updateSyncMatrix() + s.heapUpdated = false + } + syncWidth(s.pMatrix) + syncWidth(s.aMatrix) + + for i := 0; i < s.bHeap.Len(); i++ { + bar := (*s.bHeap)[i] + go bar.render(s.debugOut, tw) + } + + if err := s.flush(s.bHeap.Len()); err != nil { + fmt.Fprintf(s.debugOut, "%s %s %v\n", "[mpb]", time.Now(), err) + } +} + +func (s *pState) flush(lineCount int) error { + for s.bHeap.Len() > 0 { + bar := heap.Pop(s.bHeap).(*Bar) + frameReader := <-bar.frameReaderCh + defer func() { + if frameReader.toShutdown { + // shutdown at next flush, in other words decrement underlying WaitGroup + // only after the bar with completed state has been flushed. + // this ensures no bar ends up with less than 100% rendered. + s.shutdownPending = append(s.shutdownPending, bar) + if replacementBar, ok := s.waitBars[bar]; ok { + heap.Push(s.bHeap, replacementBar) + s.heapUpdated = true + delete(s.waitBars, bar) + } + if frameReader.removeOnComplete { + s.heapUpdated = true + return + } + } + heap.Push(s.bHeap, bar) + }() + s.cw.ReadFrom(frameReader) + lineCount += frameReader.extendedLines + } + + for i := len(s.shutdownPending) - 1; i >= 0; i-- { + close(s.shutdownPending[i].shutdown) + s.shutdownPending = s.shutdownPending[:i] + } + + return s.cw.Flush(lineCount) +} + +func syncWidth(matrix map[int][]chan int) { + for _, column := range matrix { + column := column + go func() { + var maxWidth int + for _, ch := range column { + w := <-ch + if w > maxWidth { + maxWidth = w + } + } + for _, ch := range column { + ch <- maxWidth + } + }() + } +} diff --git a/vendor/github.com/vbauerster/mpb/progress_posix.go b/vendor/github.com/vbauerster/mpb/progress_posix.go new file mode 100644 index 000000000..545245a42 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/progress_posix.go @@ -0,0 +1,70 @@ +// +build !windows + +package mpb + +import ( + "os" + "os/signal" + "syscall" + "time" +) + +func (p *Progress) serve(s *pState) { + + var ticker *time.Ticker + var refreshCh <-chan time.Time + var winch chan os.Signal + var resumeTimer *time.Timer + var resumeEvent <-chan time.Time + winchIdleDur := s.rr * 2 + + if s.manualRefreshCh == nil { + ticker = time.NewTicker(s.rr) + refreshCh = ticker.C + winch = make(chan os.Signal, 2) + signal.Notify(winch, syscall.SIGWINCH) + } else { + refreshCh = s.manualRefreshCh + } + + for { + select { + case op := <-p.operateState: + op(s) + case <-refreshCh: + if s.zeroWait { + if s.manualRefreshCh == nil { + signal.Stop(winch) + ticker.Stop() + } + if s.shutdownNotifier != nil { + close(s.shutdownNotifier) + } + close(p.done) + return + } + tw, err := s.cw.GetWidth() + if err != nil { + tw = s.width + } + s.render(tw) + case <-winch: + tw, err := s.cw.GetWidth() + if err != nil { + tw = s.width + } + s.render(tw - tw/8) + if resumeTimer != nil && resumeTimer.Reset(winchIdleDur) { + break + } + ticker.Stop() + resumeTimer = time.NewTimer(winchIdleDur) + resumeEvent = resumeTimer.C + case <-resumeEvent: + ticker = time.NewTicker(s.rr) + refreshCh = ticker.C + resumeEvent = nil + resumeTimer = nil + } + } +} diff --git a/vendor/github.com/vbauerster/mpb/progress_windows.go b/vendor/github.com/vbauerster/mpb/progress_windows.go new file mode 100644 index 000000000..cab03d36c --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/progress_windows.go @@ -0,0 +1,43 @@ +// +build windows + +package mpb + +import ( + "time" +) + +func (p *Progress) serve(s *pState) { + + var ticker *time.Ticker + var refreshCh <-chan time.Time + + if s.manualRefreshCh == nil { + ticker = time.NewTicker(s.rr) + refreshCh = ticker.C + } else { + refreshCh = s.manualRefreshCh + } + + for { + select { + case op := <-p.operateState: + op(s) + case <-refreshCh: + if s.zeroWait { + if s.manualRefreshCh == nil { + ticker.Stop() + } + if s.shutdownNotifier != nil { + close(s.shutdownNotifier) + } + close(p.done) + return + } + tw, err := s.cw.GetWidth() + if err != nil { + tw = s.width + } + s.render(tw) + } + } +} diff --git a/vendor/github.com/vbauerster/mpb/proxyreader.go b/vendor/github.com/vbauerster/mpb/proxyreader.go new file mode 100644 index 000000000..d2692ccf4 --- /dev/null +++ b/vendor/github.com/vbauerster/mpb/proxyreader.go @@ -0,0 +1,22 @@ +package mpb + +import ( + "io" + "time" +) + +// proxyReader is io.Reader wrapper, for proxy read bytes +type proxyReader struct { + io.ReadCloser + bar *Bar + iT time.Time +} + +func (pr *proxyReader) Read(p []byte) (n int, err error) { + n, err = pr.ReadCloser.Read(p) + if n > 0 { + pr.bar.IncrBy(n, time.Since(pr.iT)) + pr.iT = time.Now() + } + return +} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE b/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE deleted file mode 100644 index 511970333..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/LICENSE +++ /dev/null @@ -1,12 +0,0 @@ -Copyright (c) 2012-2015, Sergey Cherepanov -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/README.md b/vendor/gopkg.in/cheggaaa/pb.v1/README.md deleted file mode 100644 index 1295df70e..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Terminal progress bar for Go - -Simple progress bar for console programs. - -Please check the new version https://github.com/cheggaaa/pb/tree/v2 (currently, it's beta) - -## Installation - -``` -go get gopkg.in/cheggaaa/pb.v1 -``` - -## Usage - -```Go -package main - -import ( - "gopkg.in/cheggaaa/pb.v1" - "time" -) - -func main() { - count := 100000 - bar := pb.StartNew(count) - for i := 0; i < count; i++ { - bar.Increment() - time.Sleep(time.Millisecond) - } - bar.FinishPrint("The End!") -} - -``` - -Result will be like this: - -``` -> go run test.go -37158 / 100000 [================>_______________________________] 37.16% 1m11s -``` - -## Customization - -```Go -// create bar -bar := pb.New(count) - -// refresh info every second (default 200ms) -bar.SetRefreshRate(time.Second) - -// show percents (by default already true) -bar.ShowPercent = true - -// show bar (by default already true) -bar.ShowBar = true - -// no counters -bar.ShowCounters = false - -// show "time left" -bar.ShowTimeLeft = true - -// show average speed -bar.ShowSpeed = true - -// sets the width of the progress bar -bar.SetWidth(80) - -// sets the width of the progress bar, but if terminal size smaller will be ignored -bar.SetMaxWidth(80) - -// convert output to readable format (like KB, MB) -bar.SetUnits(pb.U_BYTES) - -// and start -bar.Start() -``` - -## Progress bar for IO Operations - -```go -// create and start bar -bar := pb.New(myDataLen).SetUnits(pb.U_BYTES) -bar.Start() - -// my io.Reader -r := myReader - -// my io.Writer -w := myWriter - -// create proxy reader -reader := bar.NewProxyReader(r) - -// and copy from pb reader -io.Copy(w, reader) - -``` - -```go -// create and start bar -bar := pb.New(myDataLen).SetUnits(pb.U_BYTES) -bar.Start() - -// my io.Reader -r := myReader - -// my io.Writer -w := myWriter - -// create multi writer -writer := io.MultiWriter(w, bar) - -// and copy -io.Copy(writer, r) - -bar.Finish() -``` - -## Custom Progress Bar Look-and-feel - -```go -bar.Format("<.- >") -``` - -## Multiple Progress Bars (experimental and unstable) - -Do not print to terminal while pool is active. - -```go -package main - -import ( - "math/rand" - "sync" - "time" - - "gopkg.in/cheggaaa/pb.v1" -) - -func main() { - // create bars - first := pb.New(200).Prefix("First ") - second := pb.New(200).Prefix("Second ") - third := pb.New(200).Prefix("Third ") - // start pool - pool, err := pb.StartPool(first, second, third) - if err != nil { - panic(err) - } - // update bars - wg := new(sync.WaitGroup) - for _, bar := range []*pb.ProgressBar{first, second, third} { - wg.Add(1) - go func(cb *pb.ProgressBar) { - for n := 0; n < 200; n++ { - cb.Increment() - time.Sleep(time.Millisecond * time.Duration(rand.Intn(100))) - } - cb.Finish() - wg.Done() - }(bar) - } - wg.Wait() - // close pool - pool.Stop() -} -``` - -The result will be as follows: - -``` -$ go run example/multiple.go -First 34 / 200 [=========>---------------------------------------------] 17.00% 00m08s -Second 42 / 200 [===========>------------------------------------------] 21.00% 00m06s -Third 36 / 200 [=========>---------------------------------------------] 18.00% 00m08s -``` diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/format.go b/vendor/gopkg.in/cheggaaa/pb.v1/format.go deleted file mode 100644 index 8bb8a7a1d..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/format.go +++ /dev/null @@ -1,125 +0,0 @@ -package pb - -import ( - "fmt" - "time" -) - -type Units int - -const ( - // U_NO are default units, they represent a simple value and are not formatted at all. - U_NO Units = iota - // U_BYTES units are formatted in a human readable way (B, KiB, MiB, ...) - U_BYTES - // U_BYTES_DEC units are like U_BYTES, but base 10 (B, KB, MB, ...) - U_BYTES_DEC - // U_DURATION units are formatted in a human readable way (3h14m15s) - U_DURATION -) - -const ( - KiB = 1024 - MiB = 1048576 - GiB = 1073741824 - TiB = 1099511627776 - - KB = 1e3 - MB = 1e6 - GB = 1e9 - TB = 1e12 -) - -func Format(i int64) *formatter { - return &formatter{n: i} -} - -type formatter struct { - n int64 - unit Units - width int - perSec bool -} - -func (f *formatter) To(unit Units) *formatter { - f.unit = unit - return f -} - -func (f *formatter) Width(width int) *formatter { - f.width = width - return f -} - -func (f *formatter) PerSec() *formatter { - f.perSec = true - return f -} - -func (f *formatter) String() (out string) { - switch f.unit { - case U_BYTES: - out = formatBytes(f.n) - case U_BYTES_DEC: - out = formatBytesDec(f.n) - case U_DURATION: - out = formatDuration(f.n) - default: - out = fmt.Sprintf(fmt.Sprintf("%%%dd", f.width), f.n) - } - if f.perSec { - out += "/s" - } - return -} - -// Convert bytes to human readable string. Like 2 MiB, 64.2 KiB, 52 B -func formatBytes(i int64) (result string) { - switch { - case i >= TiB: - result = fmt.Sprintf("%.02f TiB", float64(i)/TiB) - case i >= GiB: - result = fmt.Sprintf("%.02f GiB", float64(i)/GiB) - case i >= MiB: - result = fmt.Sprintf("%.02f MiB", float64(i)/MiB) - case i >= KiB: - result = fmt.Sprintf("%.02f KiB", float64(i)/KiB) - default: - result = fmt.Sprintf("%d B", i) - } - return -} - -// Convert bytes to base-10 human readable string. Like 2 MB, 64.2 KB, 52 B -func formatBytesDec(i int64) (result string) { - switch { - case i >= TB: - result = fmt.Sprintf("%.02f TB", float64(i)/TB) - case i >= GB: - result = fmt.Sprintf("%.02f GB", float64(i)/GB) - case i >= MB: - result = fmt.Sprintf("%.02f MB", float64(i)/MB) - case i >= KB: - result = fmt.Sprintf("%.02f KB", float64(i)/KB) - default: - result = fmt.Sprintf("%d B", i) - } - return -} - -func formatDuration(n int64) (result string) { - d := time.Duration(n) - if d > time.Hour*24 { - result = fmt.Sprintf("%dd", d/24/time.Hour) - d -= (d / time.Hour / 24) * (time.Hour * 24) - } - if d > time.Hour { - result = fmt.Sprintf("%s%dh", result, d/time.Hour) - d -= d / time.Hour * time.Hour - } - m := d / time.Minute - d -= m * time.Minute - s := d / time.Second - result = fmt.Sprintf("%s%02dm%02ds", result, m, s) - return -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb.go deleted file mode 100644 index e03291b89..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pb.go +++ /dev/null @@ -1,503 +0,0 @@ -// Simple console progress bars -package pb - -import ( - "fmt" - "io" - "math" - "strings" - "sync" - "sync/atomic" - "time" - "unicode/utf8" -) - -// Current version -const Version = "1.0.27" - -const ( - // Default refresh rate - 200ms - DEFAULT_REFRESH_RATE = time.Millisecond * 200 - FORMAT = "[=>-]" -) - -// DEPRECATED -// variables for backward compatibility, from now do not work -// use pb.Format and pb.SetRefreshRate -var ( - DefaultRefreshRate = DEFAULT_REFRESH_RATE - BarStart, BarEnd, Empty, Current, CurrentN string -) - -// Create new progress bar object -func New(total int) *ProgressBar { - return New64(int64(total)) -} - -// Create new progress bar object using int64 as total -func New64(total int64) *ProgressBar { - pb := &ProgressBar{ - Total: total, - RefreshRate: DEFAULT_REFRESH_RATE, - ShowPercent: true, - ShowCounters: true, - ShowBar: true, - ShowTimeLeft: true, - ShowElapsedTime: false, - ShowFinalTime: true, - Units: U_NO, - ManualUpdate: false, - finish: make(chan struct{}), - } - return pb.Format(FORMAT) -} - -// Create new object and start -func StartNew(total int) *ProgressBar { - return New(total).Start() -} - -// Callback for custom output -// For example: -// bar.Callback = func(s string) { -// mySuperPrint(s) -// } -// -type Callback func(out string) - -type ProgressBar struct { - current int64 // current must be first member of struct (https://code.google.com/p/go/issues/detail?id=5278) - previous int64 - - Total int64 - RefreshRate time.Duration - ShowPercent, ShowCounters bool - ShowSpeed, ShowTimeLeft, ShowBar bool - ShowFinalTime, ShowElapsedTime bool - Output io.Writer - Callback Callback - NotPrint bool - Units Units - Width int - ForceWidth bool - ManualUpdate bool - AutoStat bool - - // Default width for the time box. - UnitsWidth int - TimeBoxWidth int - - finishOnce sync.Once //Guards isFinish - finish chan struct{} - isFinish bool - - startTime time.Time - startValue int64 - - changeTime time.Time - - prefix, postfix string - - mu sync.Mutex - lastPrint string - - BarStart string - BarEnd string - Empty string - Current string - CurrentN string - - AlwaysUpdate bool -} - -// Start print -func (pb *ProgressBar) Start() *ProgressBar { - pb.startTime = time.Now() - pb.startValue = atomic.LoadInt64(&pb.current) - if atomic.LoadInt64(&pb.Total) == 0 { - pb.ShowTimeLeft = false - pb.ShowPercent = false - pb.AutoStat = false - } - if !pb.ManualUpdate { - pb.Update() // Initial printing of the bar before running the bar refresher. - go pb.refresher() - } - return pb -} - -// Increment current value -func (pb *ProgressBar) Increment() int { - return pb.Add(1) -} - -// Get current value -func (pb *ProgressBar) Get() int64 { - c := atomic.LoadInt64(&pb.current) - return c -} - -// Set current value -func (pb *ProgressBar) Set(current int) *ProgressBar { - return pb.Set64(int64(current)) -} - -// Set64 sets the current value as int64 -func (pb *ProgressBar) Set64(current int64) *ProgressBar { - atomic.StoreInt64(&pb.current, current) - return pb -} - -// Add to current value -func (pb *ProgressBar) Add(add int) int { - return int(pb.Add64(int64(add))) -} - -func (pb *ProgressBar) Add64(add int64) int64 { - return atomic.AddInt64(&pb.current, add) -} - -// Set prefix string -func (pb *ProgressBar) Prefix(prefix string) *ProgressBar { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.prefix = prefix - return pb -} - -// Set postfix string -func (pb *ProgressBar) Postfix(postfix string) *ProgressBar { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.postfix = postfix - return pb -} - -// Set custom format for bar -// Example: bar.Format("[=>_]") -// Example: bar.Format("[\x00=\x00>\x00-\x00]") // \x00 is the delimiter -func (pb *ProgressBar) Format(format string) *ProgressBar { - var formatEntries []string - if utf8.RuneCountInString(format) == 5 { - formatEntries = strings.Split(format, "") - } else { - formatEntries = strings.Split(format, "\x00") - } - if len(formatEntries) == 5 { - pb.BarStart = formatEntries[0] - pb.BarEnd = formatEntries[4] - pb.Empty = formatEntries[3] - pb.Current = formatEntries[1] - pb.CurrentN = formatEntries[2] - } - return pb -} - -// Set bar refresh rate -func (pb *ProgressBar) SetRefreshRate(rate time.Duration) *ProgressBar { - pb.RefreshRate = rate - return pb -} - -// Set units -// bar.SetUnits(U_NO) - by default -// bar.SetUnits(U_BYTES) - for Mb, Kb, etc -func (pb *ProgressBar) SetUnits(units Units) *ProgressBar { - pb.Units = units - return pb -} - -// Set max width, if width is bigger than terminal width, will be ignored -func (pb *ProgressBar) SetMaxWidth(width int) *ProgressBar { - pb.Width = width - pb.ForceWidth = false - return pb -} - -// Set bar width -func (pb *ProgressBar) SetWidth(width int) *ProgressBar { - pb.Width = width - pb.ForceWidth = true - return pb -} - -// End print -func (pb *ProgressBar) Finish() { - //Protect multiple calls - pb.finishOnce.Do(func() { - close(pb.finish) - pb.write(atomic.LoadInt64(&pb.Total), atomic.LoadInt64(&pb.current)) - pb.mu.Lock() - defer pb.mu.Unlock() - switch { - case pb.Output != nil: - fmt.Fprintln(pb.Output) - case !pb.NotPrint: - fmt.Println() - } - pb.isFinish = true - }) -} - -// IsFinished return boolean -func (pb *ProgressBar) IsFinished() bool { - pb.mu.Lock() - defer pb.mu.Unlock() - return pb.isFinish -} - -// End print and write string 'str' -func (pb *ProgressBar) FinishPrint(str string) { - pb.Finish() - if pb.Output != nil { - fmt.Fprintln(pb.Output, str) - } else { - fmt.Println(str) - } -} - -// implement io.Writer -func (pb *ProgressBar) Write(p []byte) (n int, err error) { - n = len(p) - pb.Add(n) - return -} - -// implement io.Reader -func (pb *ProgressBar) Read(p []byte) (n int, err error) { - n = len(p) - pb.Add(n) - return -} - -// Create new proxy reader over bar -// Takes io.Reader or io.ReadCloser -func (pb *ProgressBar) NewProxyReader(r io.Reader) *Reader { - return &Reader{r, pb} -} - -func (pb *ProgressBar) write(total, current int64) { - pb.mu.Lock() - defer pb.mu.Unlock() - width := pb.GetWidth() - - var percentBox, countersBox, timeLeftBox, timeSpentBox, speedBox, barBox, end, out string - - // percents - if pb.ShowPercent { - var percent float64 - if total > 0 { - percent = float64(current) / (float64(total) / float64(100)) - } else { - percent = float64(current) / float64(100) - } - percentBox = fmt.Sprintf(" %6.02f%%", percent) - } - - // counters - if pb.ShowCounters { - current := Format(current).To(pb.Units).Width(pb.UnitsWidth) - if total > 0 { - totalS := Format(total).To(pb.Units).Width(pb.UnitsWidth) - countersBox = fmt.Sprintf(" %s / %s ", current, totalS) - } else { - countersBox = fmt.Sprintf(" %s / ? ", current) - } - } - - // time left - currentFromStart := current - pb.startValue - fromStart := time.Now().Sub(pb.startTime) - lastChangeTime := pb.changeTime - fromChange := lastChangeTime.Sub(pb.startTime) - - if pb.ShowElapsedTime { - timeSpentBox = fmt.Sprintf(" %s ", (fromStart/time.Second)*time.Second) - } - - select { - case <-pb.finish: - if pb.ShowFinalTime { - var left time.Duration - left = (fromStart / time.Second) * time.Second - timeLeftBox = fmt.Sprintf(" %s", left.String()) - } - default: - if pb.ShowTimeLeft && currentFromStart > 0 { - perEntry := fromChange / time.Duration(currentFromStart) - var left time.Duration - if total > 0 { - left = time.Duration(total-currentFromStart) * perEntry - left -= time.Since(lastChangeTime) - left = (left / time.Second) * time.Second - } else { - left = time.Duration(currentFromStart) * perEntry - left = (left / time.Second) * time.Second - } - if left > 0 { - timeLeft := Format(int64(left)).To(U_DURATION).String() - timeLeftBox = fmt.Sprintf(" %s", timeLeft) - } - } - } - - if len(timeLeftBox) < pb.TimeBoxWidth { - timeLeftBox = fmt.Sprintf("%s%s", strings.Repeat(" ", pb.TimeBoxWidth-len(timeLeftBox)), timeLeftBox) - } - - // speed - if pb.ShowSpeed && currentFromStart > 0 { - fromStart := time.Now().Sub(pb.startTime) - speed := float64(currentFromStart) / (float64(fromStart) / float64(time.Second)) - speedBox = " " + Format(int64(speed)).To(pb.Units).Width(pb.UnitsWidth).PerSec().String() - } - - barWidth := escapeAwareRuneCountInString(countersBox + pb.BarStart + pb.BarEnd + percentBox + timeSpentBox + timeLeftBox + speedBox + pb.prefix + pb.postfix) - // bar - if pb.ShowBar { - size := width - barWidth - if size > 0 { - if total > 0 { - curSize := int(math.Ceil((float64(current) / float64(total)) * float64(size))) - emptySize := size - curSize - barBox = pb.BarStart - if emptySize < 0 { - emptySize = 0 - } - if curSize > size { - curSize = size - } - - cursorLen := escapeAwareRuneCountInString(pb.Current) - if emptySize <= 0 { - barBox += strings.Repeat(pb.Current, curSize/cursorLen) - } else if curSize > 0 { - cursorEndLen := escapeAwareRuneCountInString(pb.CurrentN) - cursorRepetitions := (curSize - cursorEndLen) / cursorLen - barBox += strings.Repeat(pb.Current, cursorRepetitions) - barBox += pb.CurrentN - } - - emptyLen := escapeAwareRuneCountInString(pb.Empty) - barBox += strings.Repeat(pb.Empty, emptySize/emptyLen) - barBox += pb.BarEnd - } else { - pos := size - int(current)%int(size) - barBox = pb.BarStart - if pos-1 > 0 { - barBox += strings.Repeat(pb.Empty, pos-1) - } - barBox += pb.Current - if size-pos-1 > 0 { - barBox += strings.Repeat(pb.Empty, size-pos-1) - } - barBox += pb.BarEnd - } - } - } - - // check len - out = pb.prefix + timeSpentBox + countersBox + barBox + percentBox + speedBox + timeLeftBox + pb.postfix - - if cl := escapeAwareRuneCountInString(out); cl < width { - end = strings.Repeat(" ", width-cl) - } - - // and print! - pb.lastPrint = out + end - isFinish := pb.isFinish - - switch { - case isFinish: - return - case pb.Output != nil: - fmt.Fprint(pb.Output, "\r"+out+end) - case pb.Callback != nil: - pb.Callback(out + end) - case !pb.NotPrint: - fmt.Print("\r" + out + end) - } -} - -// GetTerminalWidth - returns terminal width for all platforms. -func GetTerminalWidth() (int, error) { - return terminalWidth() -} - -func (pb *ProgressBar) GetWidth() int { - if pb.ForceWidth { - return pb.Width - } - - width := pb.Width - termWidth, _ := terminalWidth() - if width == 0 || termWidth <= width { - width = termWidth - } - - return width -} - -// Write the current state of the progressbar -func (pb *ProgressBar) Update() { - c := atomic.LoadInt64(&pb.current) - p := atomic.LoadInt64(&pb.previous) - t := atomic.LoadInt64(&pb.Total) - if p != c { - pb.mu.Lock() - pb.changeTime = time.Now() - pb.mu.Unlock() - atomic.StoreInt64(&pb.previous, c) - } - pb.write(t, c) - if pb.AutoStat { - if c == 0 { - pb.startTime = time.Now() - pb.startValue = 0 - } else if c >= t && pb.isFinish != true { - pb.Finish() - } - } -} - -// String return the last bar print -func (pb *ProgressBar) String() string { - pb.mu.Lock() - defer pb.mu.Unlock() - return pb.lastPrint -} - -// SetTotal atomically sets new total count -func (pb *ProgressBar) SetTotal(total int) *ProgressBar { - return pb.SetTotal64(int64(total)) -} - -// SetTotal64 atomically sets new total count -func (pb *ProgressBar) SetTotal64(total int64) *ProgressBar { - atomic.StoreInt64(&pb.Total, total) - return pb -} - -// Reset bar and set new total count -// Does effect only on finished bar -func (pb *ProgressBar) Reset(total int) *ProgressBar { - pb.mu.Lock() - defer pb.mu.Unlock() - if pb.isFinish { - pb.SetTotal(total).Set(0) - atomic.StoreInt64(&pb.previous, 0) - } - return pb -} - -// Internal loop for refreshing the progressbar -func (pb *ProgressBar) refresher() { - for { - select { - case <-pb.finish: - return - case <-time.After(pb.RefreshRate): - pb.Update() - } - } -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_appengine.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_appengine.go deleted file mode 100644 index 17168f39a..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pb_appengine.go +++ /dev/null @@ -1,11 +0,0 @@ -// +build appengine js - -package pb - -import "errors" - -// terminalWidth returns width of the terminal, which is not supported -// and should always failed on appengine classic which is a sandboxed PaaS. -func terminalWidth() (int, error) { - return 0, errors.New("Not supported") -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go deleted file mode 100644 index 9595e8236..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pb_win.go +++ /dev/null @@ -1,143 +0,0 @@ -// +build windows - -package pb - -import ( - "errors" - "fmt" - "os" - "sync" - "syscall" - "unsafe" -) - -var tty = os.Stdin - -var ( - kernel32 = syscall.NewLazyDLL("kernel32.dll") - - // GetConsoleScreenBufferInfo retrieves information about the - // specified console screen buffer. - // http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx - procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") - - // GetConsoleMode retrieves the current input mode of a console's - // input buffer or the current output mode of a console screen buffer. - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms683167(v=vs.85).aspx - getConsoleMode = kernel32.NewProc("GetConsoleMode") - - // SetConsoleMode sets the input mode of a console's input buffer - // or the output mode of a console screen buffer. - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx - setConsoleMode = kernel32.NewProc("SetConsoleMode") - - // SetConsoleCursorPosition sets the cursor position in the - // specified console screen buffer. - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms686025(v=vs.85).aspx - setConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") -) - -type ( - // Defines the coordinates of the upper left and lower right corners - // of a rectangle. - // See - // http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx - smallRect struct { - Left, Top, Right, Bottom int16 - } - - // Defines the coordinates of a character cell in a console screen - // buffer. The origin of the coordinate system (0,0) is at the top, left cell - // of the buffer. - // See - // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx - coordinates struct { - X, Y int16 - } - - word int16 - - // Contains information about a console screen buffer. - // http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx - consoleScreenBufferInfo struct { - dwSize coordinates - dwCursorPosition coordinates - wAttributes word - srWindow smallRect - dwMaximumWindowSize coordinates - } -) - -// terminalWidth returns width of the terminal. -func terminalWidth() (width int, err error) { - var info consoleScreenBufferInfo - _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0) - if e != 0 { - return 0, error(e) - } - return int(info.dwSize.X) - 1, nil -} - -func getCursorPos() (pos coordinates, err error) { - var info consoleScreenBufferInfo - _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0) - if e != 0 { - return info.dwCursorPosition, error(e) - } - return info.dwCursorPosition, nil -} - -func setCursorPos(pos coordinates) error { - _, _, e := syscall.Syscall(setConsoleCursorPosition.Addr(), 2, uintptr(syscall.Stdout), uintptr(uint32(uint16(pos.Y))<<16|uint32(uint16(pos.X))), 0) - if e != 0 { - return error(e) - } - return nil -} - -var ErrPoolWasStarted = errors.New("Bar pool was started") - -var echoLocked bool -var echoLockMutex sync.Mutex - -var oldState word - -func lockEcho() (shutdownCh chan struct{}, err error) { - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - if echoLocked { - err = ErrPoolWasStarted - return - } - echoLocked = true - - if _, _, e := syscall.Syscall(getConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&oldState)), 0); e != 0 { - err = fmt.Errorf("Can't get terminal settings: %v", e) - return - } - - newState := oldState - const ENABLE_ECHO_INPUT = 0x0004 - const ENABLE_LINE_INPUT = 0x0002 - newState = newState & (^(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT)) - if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(newState), 0); e != 0 { - err = fmt.Errorf("Can't set terminal settings: %v", e) - return - } - - shutdownCh = make(chan struct{}) - return -} - -func unlockEcho() (err error) { - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - if !echoLocked { - return - } - echoLocked = false - if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(oldState), 0); e != 0 { - err = fmt.Errorf("Can't set terminal settings") - } - return -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go b/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go deleted file mode 100644 index af4251760..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pb_x.go +++ /dev/null @@ -1,118 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd solaris dragonfly -// +build !appengine !js - -package pb - -import ( - "errors" - "fmt" - "os" - "os/signal" - "sync" - "syscall" - - "golang.org/x/sys/unix" -) - -var ErrPoolWasStarted = errors.New("Bar pool was started") - -var ( - echoLockMutex sync.Mutex - origTermStatePtr *unix.Termios - tty *os.File - istty bool -) - -func init() { - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - - var err error - tty, err = os.Open("/dev/tty") - istty = true - if err != nil { - tty = os.Stdin - istty = false - } -} - -// terminalWidth returns width of the terminal. -func terminalWidth() (int, error) { - if !istty { - return 0, errors.New("Not Supported") - } - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - - fd := int(tty.Fd()) - - ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ) - if err != nil { - return 0, err - } - - return int(ws.Col), nil -} - -func lockEcho() (shutdownCh chan struct{}, err error) { - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - if istty { - if origTermStatePtr != nil { - return shutdownCh, ErrPoolWasStarted - } - - fd := int(tty.Fd()) - - origTermStatePtr, err = unix.IoctlGetTermios(fd, ioctlReadTermios) - if err != nil { - return nil, fmt.Errorf("Can't get terminal settings: %v", err) - } - - oldTermios := *origTermStatePtr - newTermios := oldTermios - newTermios.Lflag &^= syscall.ECHO - newTermios.Lflag |= syscall.ICANON | syscall.ISIG - newTermios.Iflag |= syscall.ICRNL - if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, &newTermios); err != nil { - return nil, fmt.Errorf("Can't set terminal settings: %v", err) - } - - } - shutdownCh = make(chan struct{}) - go catchTerminate(shutdownCh) - return -} - -func unlockEcho() error { - echoLockMutex.Lock() - defer echoLockMutex.Unlock() - if istty { - if origTermStatePtr == nil { - return nil - } - - fd := int(tty.Fd()) - - if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, origTermStatePtr); err != nil { - return fmt.Errorf("Can't set terminal settings: %v", err) - } - - } - origTermStatePtr = nil - - return nil -} - -// listen exit signals and restore terminal state -func catchTerminate(shutdownCh chan struct{}) { - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL) - defer signal.Stop(sig) - select { - case <-shutdownCh: - unlockEcho() - case <-sig: - unlockEcho() - } -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool.go deleted file mode 100644 index 392e7599c..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pool.go +++ /dev/null @@ -1,104 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows - -package pb - -import ( - "io" - "sync" - "time" -) - -// Create and start new pool with given bars -// You need call pool.Stop() after work -func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) { - pool = new(Pool) - if err = pool.Start(); err != nil { - return - } - pool.Add(pbs...) - return -} - -// NewPool initialises a pool with progress bars, but -// doesn't start it. You need to call Start manually -func NewPool(pbs ...*ProgressBar) (pool *Pool) { - pool = new(Pool) - pool.Add(pbs...) - return -} - -type Pool struct { - Output io.Writer - RefreshRate time.Duration - bars []*ProgressBar - lastBarsCount int - shutdownCh chan struct{} - workerCh chan struct{} - m sync.Mutex - finishOnce sync.Once -} - -// Add progress bars. -func (p *Pool) Add(pbs ...*ProgressBar) { - p.m.Lock() - defer p.m.Unlock() - for _, bar := range pbs { - bar.ManualUpdate = true - bar.NotPrint = true - bar.Start() - p.bars = append(p.bars, bar) - } -} - -func (p *Pool) Start() (err error) { - p.RefreshRate = DefaultRefreshRate - p.shutdownCh, err = lockEcho() - if err != nil { - return - } - p.workerCh = make(chan struct{}) - go p.writer() - return -} - -func (p *Pool) writer() { - var first = true - defer func() { - if first == false { - p.print(false) - } else { - p.print(true) - p.print(false) - } - close(p.workerCh) - }() - - for { - select { - case <-time.After(p.RefreshRate): - if p.print(first) { - p.print(false) - return - } - first = false - case <-p.shutdownCh: - return - } - } -} - -// Restore terminal state and close pool -func (p *Pool) Stop() error { - p.finishOnce.Do(func() { - if p.shutdownCh != nil { - close(p.shutdownCh) - } - }) - - // Wait for the worker to complete - select { - case <-p.workerCh: - } - - return unlockEcho() -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go deleted file mode 100644 index 63598d378..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pool_win.go +++ /dev/null @@ -1,45 +0,0 @@ -// +build windows - -package pb - -import ( - "fmt" - "log" -) - -func (p *Pool) print(first bool) bool { - p.m.Lock() - defer p.m.Unlock() - var out string - if !first { - coords, err := getCursorPos() - if err != nil { - log.Panic(err) - } - coords.Y -= int16(p.lastBarsCount) - if coords.Y < 0 { - coords.Y = 0 - } - coords.X = 0 - - err = setCursorPos(coords) - if err != nil { - log.Panic(err) - } - } - isFinished := true - for _, bar := range p.bars { - if !bar.IsFinished() { - isFinished = false - } - bar.Update() - out += fmt.Sprintf("\r%s\n", bar.String()) - } - if p.Output != nil { - fmt.Fprint(p.Output, out) - } else { - fmt.Print(out) - } - p.lastBarsCount = len(p.bars) - return isFinished -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go b/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go deleted file mode 100644 index a8ae14d2f..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/pool_x.go +++ /dev/null @@ -1,29 +0,0 @@ -// +build linux darwin freebsd netbsd openbsd solaris dragonfly - -package pb - -import "fmt" - -func (p *Pool) print(first bool) bool { - p.m.Lock() - defer p.m.Unlock() - var out string - if !first { - out = fmt.Sprintf("\033[%dA", p.lastBarsCount) - } - isFinished := true - for _, bar := range p.bars { - if !bar.IsFinished() { - isFinished = false - } - bar.Update() - out += fmt.Sprintf("\r%s\n", bar.String()) - } - if p.Output != nil { - fmt.Fprint(p.Output, out) - } else { - fmt.Print(out) - } - p.lastBarsCount = len(p.bars) - return isFinished -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/reader.go b/vendor/gopkg.in/cheggaaa/pb.v1/reader.go deleted file mode 100644 index 9f3148b54..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/reader.go +++ /dev/null @@ -1,25 +0,0 @@ -package pb - -import ( - "io" -) - -// It's proxy reader, implement io.Reader -type Reader struct { - io.Reader - bar *ProgressBar -} - -func (r *Reader) Read(p []byte) (n int, err error) { - n, err = r.Reader.Read(p) - r.bar.Add(n) - return -} - -// Close the reader when it implements io.Closer -func (r *Reader) Close() (err error) { - if closer, ok := r.Reader.(io.Closer); ok { - return closer.Close() - } - return -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go b/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go deleted file mode 100644 index c617c55ec..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/runecount.go +++ /dev/null @@ -1,17 +0,0 @@ -package pb - -import ( - "github.com/mattn/go-runewidth" - "regexp" -) - -// Finds the control character sequences (like colors) -var ctrlFinder = regexp.MustCompile("\x1b\x5b[0-9]+\x6d") - -func escapeAwareRuneCountInString(s string) int { - n := runewidth.StringWidth(s) - for _, sm := range ctrlFinder.FindAllString(s, -1) { - n -= runewidth.StringWidth(sm) - } - return n -} diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go b/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go deleted file mode 100644 index 517ea8ed7..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/termios_bsd.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build darwin freebsd netbsd openbsd dragonfly -// +build !appengine - -package pb - -import "syscall" - -const ioctlReadTermios = syscall.TIOCGETA -const ioctlWriteTermios = syscall.TIOCSETA diff --git a/vendor/gopkg.in/cheggaaa/pb.v1/termios_sysv.go b/vendor/gopkg.in/cheggaaa/pb.v1/termios_sysv.go deleted file mode 100644 index b10f61859..000000000 --- a/vendor/gopkg.in/cheggaaa/pb.v1/termios_sysv.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build linux solaris -// +build !appengine - -package pb - -import "golang.org/x/sys/unix" - -const ioctlReadTermios = unix.TCGETS -const ioctlWriteTermios = unix.TCSETS |