diff options
116 files changed, 2272 insertions, 1058 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 625b96fdd..09f13a7d0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,48 +5,118 @@ # and storage. gcp_credentials: ENCRYPTED[885c6e4297dd8d6f67593c42b810353af0c505a7a670e2c6fd830c56e86bbb2debcc3c18f942d0d46ab36b63521061d4] -# Default VM to use for testing, unless values overriden by specific tasks (below) -gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 40 - -# Main collection of env. varss to set for all scripts. All others -# are cooked in by $SCRIPT_BASE/setup_environment.sh +# Default timeout for each task +timeout_in: 120m + +# Main collection of env. vars to set for all tasks and scripts. env: - FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" - CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" - CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" - CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" - RUNC_COMMIT: "96ec2177ae841256168fcf76954f7177af9446eb" + #### + #### Global variables used for all tasks + #### # File to update in home-dir with task-specific env. var values ENVLIB: ".bash_profile" # Overrides default location (/tmp/cirrus) for repo clone - CIRRUS_WORKING_DIR: "/go/src/github.com/containers/libpod" + CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod" # Required so $ENVLIB gets loaded CIRRUS_SHELL: "/bin/bash" # Save a little typing (path relative to $CIRRUS_WORKING_DIR) SCRIPT_BASE: "./contrib/cirrus" PACKER_BASE: "./contrib/cirrus/packer" + + #### + #### Variables for composing new cache-images (used in PR testing) from + #### base-images (pre-existing in GCE) + #### + # Git commits to use while building dependencies into cache-images + FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" + CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" + CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" + CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" + RUNC_COMMIT: "25f3f893c86d07426df93b7aa172f33fdf093fbd" + # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) + PACKER_BUILDS: "ubuntu-18,fedora-29" # TODO: fah-29,rhel-7,centos-7 + # Version of packer to use + PACKER_VER: "1.3.1" + # Google-maintained base-image names + UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a" + CENTOS_BASE_IMAGE: "centos-7-v20181113" + # Manually produced base-image names (see $SCRIPT_BASE/README.md) + FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245" + FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861" + # RHEL image must be imported, google bills extra for their native image. + RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1541783972" + + #### + #### Credentials and other secret-sauces, decrypted at runtime when authorized. + #### + # Freenode IRC credentials for posting status messages IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883] + # Command to register a RHEL VM to install/update packages + RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] + # Needed to build GCE images, within a GCE VM + SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] + # User ID for cirrus to ssh into VMs + GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277] + # Name where this repositories cloud resources are located + GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + + # Space separated list of environment variables to unset before testing + UNSET_ENV_VARS: >- + GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX + IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE + UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT + CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE + CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI + CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA + CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH + CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST + CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME + CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME + CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION + CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX + XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID + +# Every *_task runs in parallel in separate VMsd. The name prefix only for reference +# in WebUI, and will be followed by matrix details. This task gates all others with +# quick format, lint, and unit tests on the standard platform. +gating_task: + + env: + CIRRUS_WORKING_DIR: "/usr/src/libpod" + + # Runs within Cirrus's "community cluster" + container: + image: "quay.io/libpod/gate:latest" + cpu: 4 + memory: 12 + + gate_script: + - '/usr/local/bin/entrypoint.sh validate' + - '/usr/local/bin/entrypoint.sh lint' + -# Every *_task runs in parallel in separate VMs. The name prefix only for reference -# in WebUI, and will be followed by matrix details. This task does all the -# per-pr unit/integration testing. -full_vm_testing_task: +# This task does the unit and integration testing for every platform +testing_task: + + depends_on: + - "gating" gce_instance: - # Generate multiple 'test' tasks, covering all possible - # 'matrix' combinations. All run in parallel. + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 2 + memory: "4Gb" + disk: 40 + # Generate multiple parallel tasks, covering all possible + # 'matrix' combinations. matrix: - # Images are generated separetly, from build_images_task (below) + # Images are generated separately, from build_images_task (below) image_name: "ubuntu-18-libpod-0c954a67" - # TODO: Make these work (also build_images_task below) - #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" - #image_name: "centos-7-v20180911-libpod-fce09afe" - #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" + image_name: "fedora-29-libpod-0c954a67" + # TODO: tests fail + # image_name: "rhel-7-something-something" + # image_name: "centos-7-something-something" + # image_name: "fah-29-libpod-5070733157859328" timeout_in: 120m @@ -55,11 +125,9 @@ full_vm_testing_task: setup_environment_script: $SCRIPT_BASE/setup_environment.sh # ...or lists of strings - verify_source_script: - - whoami # root! - - $SCRIPT_BASE/verify_source.sh - - unit_test_script: $SCRIPT_BASE/unit_test.sh + unit_test_script: + - go version + - $SCRIPT_BASE/unit_test.sh integration_test_script: $SCRIPT_BASE/integration_test.sh @@ -68,8 +136,7 @@ full_vm_testing_task: # Because system tests are stored within the repository, it is sometimes # necessary to execute them within a PR to validate changes. - -optional_system_testing_task: +optional_testing_task: # Only run system tests in PRs (not on merge) if magic string is present # in the PR description. Post-merge system testing is assumed to happen @@ -79,12 +146,13 @@ optional_system_testing_task: $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' gce_instance: + image_project: "libpod-218412" matrix: - image_name: "ubuntu-1804-bionic-v20180911-libpod-e8d18305" + image_name: "ubuntu-18-libpod-0c954a67" + image_name: "fedora-29-libpod-0c954a67" # TODO: Make these work (also build_images_task below) #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" #image_name: "centos-7-v20180911-libpod-fce09afe" - #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" timeout_in: 60m @@ -93,13 +161,11 @@ optional_system_testing_task: success_script: $SCRIPT_BASE/success.sh -# This task builds new cache-images for future PR testing. These images save -# time installing/setting up the environment while an engineer is waiting. -# The 'active' cache-images for full_vm_testing are selected by the -# 'image_name' keys. Updating those items requires manually modification, -# but this could be automated (see comment at end of build_vm_images_task). - -build_vm_images_task: +# Build new cache-images for future PR testing, but only after a PR merge. +# The cache-images save install/setup time needed test every PR. The 'active' images +# are selected by the 'image_name' items tasks above. Currently this requires +# manually updating the names, but this could be automated (see comment below). +cache_images_task: # Only produce new cache-images after a PR merge, and if a magic string # is present in the most recent commit-message. only_if: >- @@ -108,45 +174,27 @@ build_vm_images_task: # Require tests to pass first. depends_on: - - full_vm_testing # i.e. 'full_vm_testing_task' - - env: - # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json) - PACKER_BUILDS: "ubuntu-18" - # TODO: Make these work (also full_vm_testing_task above) - # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18" - CENTOS_BASE_IMAGE: "centos-7-v20180911" - RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1" - FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7" - UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911" - # low-level base VM image name inputs to packer - - # Command to register a RHEL VM - RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] - # Additional environment variables needed to build GCE images, within a GCE VM - SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] - GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277] - GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - # Version of packer to use - PACKER_VER: "1.3.1" + - "gating" + - "testing" # VMs created by packer are not cleaned up by cirrus auto_cancellation: $CI != "true" gce_instance: - image_name: "image-builder-image" # Simply CentOS 7 + packer dependencies + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 4 + memory: "4Gb" + disk: 20 + image_name: "image-builder-image-1541772081" # Simply CentOS 7 + packer dependencies # Additional permissions for building GCE images, within a GCE VM scopes: - compute - devstorage.full_control - # Doesn't need many local resources to run - cpu: 2 - memory: "2Gb" - disk: 20 environment_script: $SCRIPT_BASE/setup_environment.sh build_vm_images_script: $SCRIPT_BASE/build_vm_images.sh - # TODO,Continuous Delivery: Automaticly open a libpod PR after using 'sed' to replace + # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace # the image_names with the new (just build) images. That will # cause a new round of testing to happen (via the PR) using # the new images. When all is good, the PR may be manually @@ -113,7 +113,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func StopContainer(name: string, timeout: int) string](#StopContainer) -[func StopPod(name: string) string](#StopPod) +[func StopPod(name: string, timeout: int) string](#StopPod) [func TagImage(name: string, tagged: string) string](#TagImage) @@ -609,7 +609,8 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.PullImage '{"name": "regi method PushImage(name: [string](https://godoc.org/builtin#string), tag: [string](https://godoc.org/builtin#string), tlsverify: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div> PushImage takes three input arguments: the name or ID of an image, the fully-qualified destination name of the image, -and a boolean as to whether tls-verify should be used. It will return an [ImageNotFound](#ImageNotFound) error if +and a boolean as to whether tls-verify should be used (with false disabling TLS, not affecting the default behavior). +It will return an [ImageNotFound](#ImageNotFound) error if the image cannot be found in local storage; otherwise the ID of the image will be returned on success. ### <a name="RemoveContainer"></a>func RemoveContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -740,8 +741,8 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.StopContainer '{"name": " ### <a name="StopPod"></a>func StopPod <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> -method StopPod(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> -StopPod stops containers in a pod. It takes the name or ID of a pod. +method StopPod(name: [string](https://godoc.org/builtin#string), timeout: [int](https://godoc.org/builtin#int)) [string](https://godoc.org/builtin#string)</div> +StopPod stops containers in a pod. It takes the name or ID of a pod and a timeout. If the pod cannot be found, a [PodNotFound](#PodNotFound) error will be returned instead. Containers in a pod are stopped independently. If there is an error stopping one container, the ID of those containers will be returned in a list, along with the ID of the pod in a [PodContainerError](#PodContainerError). @@ -172,6 +172,7 @@ testunit: libpodimage localunit: test/goecho/goecho varlink_generate $(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES) + $(MAKE) -C contrib/cirrus/packer test ginkgo: ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/. diff --git a/cmd/podman/create.go b/cmd/podman/create.go index 228438d75..6c6bcfb41 100644 --- a/cmd/podman/create.go +++ b/cmd/podman/create.go @@ -129,7 +129,7 @@ func createContainer(c *cli.Context, runtime *libpod.Runtime) (*libpod.Container var data *inspect.ImageData = nil if rootfs == "" && !rootless.SkipStorageSetup() { - newImage, err := runtime.ImageRuntime().New(ctx, c.Args()[0], rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false, false) + newImage, err := runtime.ImageRuntime().New(ctx, c.Args()[0], rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false) if err != nil { return nil, nil, err } diff --git a/cmd/podman/generate.go b/cmd/podman/generate.go new file mode 100644 index 000000000..765d0ee70 --- /dev/null +++ b/cmd/podman/generate.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/urfave/cli" +) + +var ( + generateSubCommands = []cli.Command{ + containerKubeCommand, + } + + generateDescription = "generate structured data based for a containers and pods" + kubeCommand = cli.Command{ + Name: "generate", + Usage: "generated structured data", + Description: generateDescription, + ArgsUsage: "", + Subcommands: generateSubCommands, + UseShortOptionHandling: true, + OnUsageError: usageErrorHandler, + Hidden: true, + } +) diff --git a/cmd/podman/kube_generate.go b/cmd/podman/generate_kube.go index a18912668..de9f701b0 100644 --- a/cmd/podman/kube_generate.go +++ b/cmd/podman/generate_kube.go @@ -6,10 +6,11 @@ import ( "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/libpod" "github.com/containers/libpod/pkg/rootless" + podmanVersion "github.com/containers/libpod/version" "github.com/ghodss/yaml" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/urfave/cli" + "k8s.io/api/core/v1" ) var ( @@ -18,16 +19,15 @@ var ( Name: "service, s", Usage: "only generate YAML for kubernetes service object", }, - LatestFlag, } containerKubeDescription = "Generate Kubernetes Pod YAML" containerKubeCommand = cli.Command{ - Name: "generate", - Usage: "Generate Kubernetes pod YAML for a container", + Name: "kube", + Usage: "Generate Kubernetes pod YAML for a container or pod", Description: containerKubeDescription, Flags: sortFlags(containerKubeFlags), Action: generateKubeYAMLCmd, - ArgsUsage: "CONTAINER-NAME", + ArgsUsage: "CONTAINER|POD-NAME", UseShortOptionHandling: true, OnUsageError: usageErrorHandler, } @@ -36,9 +36,13 @@ var ( // generateKubeYAMLCmdgenerates or replays kube func generateKubeYAMLCmd(c *cli.Context) error { var ( - container *libpod.Container - err error - output []byte + podYAML *v1.Pod + container *libpod.Container + err error + output []byte + pod *libpod.Pod + mashalledBytes []byte + servicePorts []v1.ServicePort ) if rootless.IsRootless() { @@ -46,10 +50,7 @@ func generateKubeYAMLCmd(c *cli.Context) error { } args := c.Args() if len(args) > 1 || (len(args) < 1 && !c.Bool("latest")) { - return errors.Errorf("you must provide one container ID or name or --latest") - } - if c.Bool("service") { - return errors.Wrapf(libpod.ErrNotImplemented, "service generation") + return errors.Errorf("you must provide one container|pod ID or name or --latest") } runtime, err := libpodruntime.GetRuntime(c) @@ -59,33 +60,43 @@ func generateKubeYAMLCmd(c *cli.Context) error { defer runtime.Shutdown(false) // Get the container in question - if c.Bool("latest") { - container, err = runtime.GetLatestContainer() + container, err = runtime.LookupContainer(args[0]) + if err != nil { + pod, err = runtime.LookupPod(args[0]) + if err != nil { + return err + } + podYAML, servicePorts, err = pod.GenerateForKube() } else { - container, err = runtime.LookupContainer(args[0]) + if len(container.Dependencies()) > 0 { + return errors.Wrapf(libpod.ErrNotImplemented, "containers with dependencies") + } + podYAML, err = container.GenerateForKube() } if err != nil { return err } - if len(container.Dependencies()) > 0 { - return errors.Wrapf(libpod.ErrNotImplemented, "containers with dependencies") + if c.Bool("service") { + serviceYAML := libpod.GenerateKubeServiceFromV1Pod(podYAML, servicePorts) + mashalledBytes, err = yaml.Marshal(serviceYAML) + } else { + // Marshall the results + mashalledBytes, err = yaml.Marshal(podYAML) } - - podYAML, err := container.InspectForKube() if err != nil { return err } - developmentComment := []byte("# Generation of Kubenetes YAML is still under development!\n") - logrus.Warn("This function is still under heavy development.") - // Marshall the results - b, err := yaml.Marshal(podYAML) - if err != nil { - return err - } - output = append(output, developmentComment...) - output = append(output, b...) + header := `# Generation of Kubenetes YAML is still under development! +# +# Save the output of this file and use kubectl create -f to import +# it into Kubernetes. +# +# Created with podman-%s +` + output = append(output, []byte(fmt.Sprintf(header, podmanVersion.Version))...) + output = append(output, mashalledBytes...) // Output the v1.Pod with the v1.Container fmt.Println(string(output)) diff --git a/cmd/podman/kube.go b/cmd/podman/kube.go deleted file mode 100644 index 2cb407c09..000000000 --- a/cmd/podman/kube.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "github.com/urfave/cli" -) - -var ( - kubeSubCommands = []cli.Command{ - containerKubeCommand, - } - - kubeDescription = "Work with Kubernetes objects" - kubeCommand = cli.Command{ - Name: "kube", - Usage: "Import and export Kubernetes objections from and to Podman", - Description: containerDescription, - ArgsUsage: "", - Subcommands: kubeSubCommands, - UseShortOptionHandling: true, - OnUsageError: usageErrorHandler, - Hidden: true, - } -) diff --git a/cmd/podman/login.go b/cmd/podman/login.go index 33ce8635f..cfdd8005b 100644 --- a/cmd/podman/login.go +++ b/cmd/podman/login.go @@ -8,6 +8,7 @@ import ( "github.com/containers/image/docker" "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/types" "github.com/containers/libpod/libpod/common" "github.com/pkg/errors" "github.com/urfave/cli" @@ -93,7 +94,9 @@ func loginCmd(c *cli.Context) error { return errors.Wrapf(err, "error getting username and password") } - sc.DockerInsecureSkipTLSVerify = !c.BoolT("tls-verify") + if c.IsSet("tls-verify") { + sc.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) + } if c.String("cert-dir") != "" { sc.DockerCertPath = c.String("cert-dir") } diff --git a/cmd/podman/pod_stop.go b/cmd/podman/pod_stop.go index 14114aa11..d49ba8a00 100644 --- a/cmd/podman/pod_stop.go +++ b/cmd/podman/pod_stop.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -16,6 +15,10 @@ var ( Usage: "stop all running pods", }, LatestPodFlag, + cli.UintFlag{ + Name: "timeout, time, t", + Usage: "Seconds to wait for pod stop before killing the container", + }, } podStopDescription = ` podman pod stop @@ -35,6 +38,7 @@ var ( ) func podStopCmd(c *cli.Context) error { + timeout := -1 if err := checkMutuallyExclusiveFlags(c); err != nil { return err } @@ -52,9 +56,12 @@ func podStopCmd(c *cli.Context) error { ctx := getContext() + if c.IsSet("timeout") { + timeout = int(c.Uint("timeout")) + } for _, pod := range pods { // set cleanup to true to clean mounts and namespaces - ctr_errs, err := pod.Stop(ctx, true) + ctr_errs, err := pod.StopWithTimeout(ctx, true, timeout) if ctr_errs != nil { for ctr, err := range ctr_errs { if lastError != nil { diff --git a/cmd/podman/pull.go b/cmd/podman/pull.go index 8fb3971bd..47130805e 100644 --- a/cmd/podman/pull.go +++ b/cmd/podman/pull.go @@ -64,7 +64,6 @@ specified, the image with the 'latest' tag (if it exists) is pulled // pullCmd gets the data from the command line and calls pullImage // to copy an image from a registry to a local machine func pullCmd(c *cli.Context) error { - forceSecure := false runtime, err := libpodruntime.GetRuntime(c) if err != nil { return errors.Wrapf(err, "could not get runtime") @@ -104,12 +103,11 @@ func pullCmd(c *cli.Context) error { } dockerRegistryOptions := image2.DockerRegistryOptions{ - DockerRegistryCreds: registryCreds, - DockerCertPath: c.String("cert-dir"), - DockerInsecureSkipTLSVerify: !c.BoolT("tls-verify"), + DockerRegistryCreds: registryCreds, + DockerCertPath: c.String("cert-dir"), } if c.IsSet("tls-verify") { - forceSecure = c.Bool("tls-verify") + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } // Possible for docker-archive to have multiple tags, so use LoadFromArchiveReference instead @@ -125,7 +123,7 @@ func pullCmd(c *cli.Context) error { imgID = newImage[0].ID() } else { authfile := getAuthFile(c.String("authfile")) - newImage, err := runtime.ImageRuntime().New(getContext(), image, c.String("signature-policy"), authfile, writer, &dockerRegistryOptions, image2.SigningOptions{}, true, forceSecure) + newImage, err := runtime.ImageRuntime().New(getContext(), image, c.String("signature-policy"), authfile, writer, &dockerRegistryOptions, image2.SigningOptions{}, true) if err != nil { return errors.Wrapf(err, "error pulling image %q", image) } diff --git a/cmd/podman/push.go b/cmd/podman/push.go index 331f92cd2..82589f3f1 100644 --- a/cmd/podman/push.go +++ b/cmd/podman/push.go @@ -81,7 +81,6 @@ func pushCmd(c *cli.Context) error { var ( registryCreds *types.DockerAuthConfig destName string - forceSecure bool ) args := c.Args() @@ -108,7 +107,6 @@ func pushCmd(c *cli.Context) error { } certPath := c.String("cert-dir") - skipVerify := !c.BoolT("tls-verify") removeSignatures := c.Bool("remove-signatures") signBy := c.String("sign-by") @@ -145,14 +143,12 @@ func pushCmd(c *cli.Context) error { } } - if c.IsSet("tls-verify") { - forceSecure = c.Bool("tls-verify") - } - dockerRegistryOptions := image.DockerRegistryOptions{ - DockerRegistryCreds: registryCreds, - DockerCertPath: certPath, - DockerInsecureSkipTLSVerify: skipVerify, + DockerRegistryCreds: registryCreds, + DockerCertPath: certPath, + } + if c.IsSet("tls-verify") { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } so := image.SigningOptions{ @@ -167,5 +163,5 @@ func pushCmd(c *cli.Context) error { authfile := getAuthFile(c.String("authfile")) - return newImage.PushImageToHeuristicDestination(getContext(), destName, manifestType, authfile, c.String("signature-policy"), writer, c.Bool("compress"), so, &dockerRegistryOptions, forceSecure, nil) + return newImage.PushImageToHeuristicDestination(getContext(), destName, manifestType, authfile, c.String("signature-policy"), writer, c.Bool("compress"), so, &dockerRegistryOptions, nil) } diff --git a/cmd/podman/runlabel.go b/cmd/podman/runlabel.go index b0d87d0d9..48a296260 100644 --- a/cmd/podman/runlabel.go +++ b/cmd/podman/runlabel.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod/image" @@ -153,8 +154,10 @@ func runlabelCmd(c *cli.Context) error { } dockerRegistryOptions := image.DockerRegistryOptions{ - DockerCertPath: c.String("cert-dir"), - DockerInsecureSkipTLSVerify: !c.BoolT("tls-verify"), + DockerCertPath: c.String("cert-dir"), + } + if c.IsSet("tls-verify") { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } authfile := getAuthFile(c.String("authfile")) diff --git a/cmd/podman/save.go b/cmd/podman/save.go index 7edc42e0d..139f3918a 100644 --- a/cmd/podman/save.go +++ b/cmd/podman/save.go @@ -146,7 +146,7 @@ func saveCmd(c *cli.Context) error { return err } } - if err := newImage.PushImageToReference(getContext(), destRef, manifestType, "", "", writer, c.Bool("compress"), libpodImage.SigningOptions{}, &libpodImage.DockerRegistryOptions{}, false, additionaltags); err != nil { + if err := newImage.PushImageToReference(getContext(), destRef, manifestType, "", "", writer, c.Bool("compress"), libpodImage.SigningOptions{}, &libpodImage.DockerRegistryOptions{}, additionaltags); err != nil { if err2 := os.Remove(output); err2 != nil { logrus.Errorf("error deleting %q: %v", output, err) } diff --git a/cmd/podman/search.go b/cmd/podman/search.go index fa11dad32..442ebb57f 100644 --- a/cmd/podman/search.go +++ b/cmd/podman/search.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containers/image/docker" + "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/formats" "github.com/containers/libpod/libpod/common" sysreg "github.com/containers/libpod/pkg/registries" @@ -72,11 +73,12 @@ type searchParams struct { } type searchOpts struct { - filter []string - limit int - noTrunc bool - format string - authfile string + filter []string + limit int + noTrunc bool + format string + authfile string + insecureSkipTLSVerify types.OptionalBool } type searchFilterParams struct { @@ -116,7 +118,10 @@ func searchCmd(c *cli.Context) error { filter: c.StringSlice("filter"), authfile: getAuthFile(c.String("authfile")), } - regAndSkipTLS, err := getRegistriesAndSkipTLS(c, registry) + if c.IsSet("tls-verify") { + opts.insecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) + } + registries, err := getRegistries(registry) if err != nil { return err } @@ -126,7 +131,7 @@ func searchCmd(c *cli.Context) error { return err } - return generateSearchOutput(term, regAndSkipTLS, opts, *filter) + return generateSearchOutput(term, registries, opts, *filter) } func genSearchFormat(format string) string { @@ -157,16 +162,8 @@ func (s *searchParams) headerMap() map[string]string { return values } -// A function for finding which registries can skip TLS -func getRegistriesAndSkipTLS(c *cli.Context, registry string) (map[string]bool, error) { - // Variables for setting up Registry and TLSVerify - tlsVerify := c.BoolT("tls-verify") - forceSecure := false - - if c.IsSet("tls-verify") { - forceSecure = c.BoolT("tls-verify") - } - +// getRegistries returns the list of registries to search, depending on an optional registry specification +func getRegistries(registry string) ([]string, error) { var registries []string if registry != "" { registries = append(registries, registry) @@ -177,35 +174,10 @@ func getRegistriesAndSkipTLS(c *cli.Context, registry string) (map[string]bool, return nil, errors.Wrapf(err, "error getting registries to search") } } - regAndSkipTLS := make(map[string]bool) - // If tls-verify is set to false, allow insecure always. - if !tlsVerify { - for _, reg := range registries { - regAndSkipTLS[reg] = true - } - } else { - // initially set all registries to verify with TLS - for _, reg := range registries { - regAndSkipTLS[reg] = false - } - // if the user didn't allow nor disallow insecure registries, check to see if the registry is insecure - if !forceSecure { - insecureRegistries, err := sysreg.GetInsecureRegistries() - if err != nil { - return nil, errors.Wrapf(err, "error getting insecure registries to search") - } - for _, reg := range insecureRegistries { - // if there are any insecure registries in registries, allow for HTTP - if _, ok := regAndSkipTLS[reg]; ok { - regAndSkipTLS[reg] = true - } - } - } - } - return regAndSkipTLS, nil + return registries, nil } -func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts, filter searchFilterParams) ([]searchParams, error) { +func getSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) ([]searchParams, error) { // Max number of queries by default is 25 limit := maxQueries if opts.limit != 0 { @@ -213,10 +185,10 @@ func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts } sc := common.GetSystemContext("", opts.authfile, false) + sc.DockerInsecureSkipTLSVerify = opts.insecureSkipTLSVerify + sc.SystemRegistriesConfPath = sysreg.SystemRegistriesConfPath() // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. var paramsArr []searchParams - for reg, skipTLS := range regAndSkipTLS { - // set the SkipTLSVerify bool depending on the registry being searched through - sc.DockerInsecureSkipTLSVerify = skipTLS + for _, reg := range registries { results, err := docker.SearchRegistry(context.TODO(), sc, reg, term, limit) if err != nil { logrus.Errorf("error searching registry %q: %v", reg, err) @@ -276,8 +248,8 @@ func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts return paramsArr, nil } -func generateSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts, filter searchFilterParams) error { - searchOutput, err := getSearchOutput(term, regAndSkipTLS, opts, filter) +func generateSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) error { + searchOutput, err := getSearchOutput(term, registries, opts, filter) if err != nil { return err } diff --git a/cmd/podman/shared/container.go b/cmd/podman/shared/container.go index 90ce193f7..6236d19b4 100644 --- a/cmd/podman/shared/container.go +++ b/cmd/podman/shared/container.go @@ -4,11 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/containers/image/types" - "github.com/containers/libpod/libpod/image" - "github.com/containers/libpod/pkg/util" - "github.com/cri-o/ocicni/pkg/ocicni" - "github.com/docker/go-units" "io" "os" "path/filepath" @@ -18,9 +13,14 @@ import ( "sync" "time" + "github.com/containers/image/types" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/inspect" cc "github.com/containers/libpod/pkg/spec" + "github.com/containers/libpod/pkg/util" + "github.com/cri-o/ocicni/pkg/ocicni" + "github.com/docker/go-units" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -620,7 +620,7 @@ func GetRunlabel(label string, runlabelImage string, ctx context.Context, runtim registryCreds = creds } dockerRegistryOptions.DockerRegistryCreds = registryCreds - newImage, err = runtime.ImageRuntime().New(ctx, runlabelImage, signaturePolicyPath, authfile, output, &dockerRegistryOptions, image.SigningOptions{}, false, false) + newImage, err = runtime.ImageRuntime().New(ctx, runlabelImage, signaturePolicyPath, authfile, output, &dockerRegistryOptions, image.SigningOptions{}, false) } else { newImage, err = runtime.ImageRuntime().NewFromLocal(runlabelImage) } diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index b081b60a3..3cdc99a83 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -610,7 +610,8 @@ method InspectImage(name: string) -> (image: string) method HistoryImage(name: string) -> (history: []ImageHistory) # PushImage takes three input arguments: the name or ID of an image, the fully-qualified destination name of the image, -# and a boolean as to whether tls-verify should be used. It will return an [ImageNotFound](#ImageNotFound) error if +# and a boolean as to whether tls-verify should be used (with false disabling TLS, not affecting the default behavior). +# It will return an [ImageNotFound](#ImageNotFound) error if # the image cannot be found in local storage; otherwise the ID of the image will be returned on success. method PushImage(name: string, tag: string, tlsverify: bool) -> (image: string) @@ -714,7 +715,7 @@ method InspectPod(name: string) -> (pod: string) # ~~~ method StartPod(name: string) -> (pod: string) -# StopPod stops containers in a pod. It takes the name or ID of a pod. +# StopPod stops containers in a pod. It takes the name or ID of a pod and a timeout. # If the pod cannot be found, a [PodNotFound](#PodNotFound) error will be returned instead. # Containers in a pod are stopped independently. If there is an error stopping one container, the ID of those containers # will be returned in a list, along with the ID of the pod in a [PodContainerError](#PodContainerError). @@ -727,7 +728,7 @@ method StartPod(name: string) -> (pod: string) # "pod": "135d71b9495f7c3967f536edad57750bfdb569336cd107d8aabab45565ffcfb6" # } # ~~~ -method StopPod(name: string) -> (pod: string) +method StopPod(name: string, timeout: int) -> (pod: string) # RestartPod will restart containers in a pod given a pod name or ID. Containers in # the pod that are running will be stopped, then all stopped containers will be run. diff --git a/commands.md b/commands.md index 8e1a2dd7a..43796722f 100644 --- a/commands.md +++ b/commands.md @@ -16,6 +16,7 @@ | [podman-diff(1)](/docs/podman-diff.1.md) | Inspect changes on a container or image's filesystem |[](https://asciinema.org/a/FXfWB9CKYFwYM4EfqW3NSZy1G)| | [podman-exec(1)](/docs/podman-exec.1.md) | Execute a command in a running container | [podman-export(1)](/docs/podman-export.1.md) | Export container's filesystem contents as a tar archive |[](https://asciinema.org/a/913lBIRAg5hK8asyIhhkQVLtV)| +| [podman-generate(1)](/docs/podman-generate.1.md) | Generate structured output based on Podman containers and pods | | | [podman-history(1)](/docs/podman-history.1.md) | Shows the history of an image |[](https://asciinema.org/a/bCvUQJ6DkxInMELZdc5DinNSx)| | [podman-image(1)](/docs/podman-image.1.md) | Manage Images|| | [podman-images(1)](/docs/podman-images.1.md) | List images in local storage |[](https://asciinema.org/a/133649)| diff --git a/completions/bash/podman b/completions/bash/podman index 3382b6c5a..9a8adb502 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -876,6 +876,25 @@ _podman_container_wait() { _podman_wait } +_podman_generate() { + local boolean_options=" + --help + -h + " + subcommands=" + kube + " + __podman_subcommands "$subcommands $aliases" && return + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + COMPREPLY=( $( compgen -W "$subcommands" -- "$cur" ) ) + ;; + esac +} _podman_container() { local boolean_options=" --help @@ -2219,6 +2238,14 @@ _podman_logout() { _complete_ "$options_with_args" "$boolean_options" } +_podman_generate_kube() { + local options_with_args="" + + local boolean_options=" + -s + --service + " + _podman_container_runlabel() { local options_with_args=" --authfile @@ -2451,6 +2478,8 @@ _podman_pod_start() { _podman_pod_stop() { local options_with_args=" + -t + --timeout " local boolean_options=" @@ -2702,6 +2731,7 @@ _podman_podman() { diff exec export + generate history images import diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index fa233a2cb..c5c976358 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -5,6 +5,7 @@ Similar to other integrated github CI/CD services, Cirrus utilizes a simple YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ + ## Workflow All tasks execute in parallel, unless there are conditions or dependencies @@ -12,24 +13,34 @@ which alter this behavior. Within each task, each script executes in sequence, so long as any previous script exited successfully. The overall state of each task (pass or fail) is set based on the exit status of the last script to execute. -### ``full_vm_testing`` Task -1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined - in ``.cirrus.yml``. Once accessible, ``ssh`` into each VM and run the following - scripts. +### ``gating`` Task + +***N/B: Steps below are performed by automation*** + +1. Launch a purpose-built container in Cirrus's community cluster. + For container image details, please see + [the contributors guide](https://github.com/containers/libpod/blob/master/CONTRIBUTING.md#go-format-and-lint). + +3. ``validate``: Perform standard `make validate` source verification, + Should run for less than a minute or two. + +4. ``lint``: Execute regular `make lint` to check for any code cruft. + Should also run for less than a few minutes. + -2. ``setup_environment.sh``: Configure root's ``.bash_profile`` - for all subsequent scripts (each run in a new shell). Any - distribution-specific environment variables are also defined - here. For example, setting tags/flags to use compiling. +### ``testing`` Task -3. ``verify_source.sh``: Perform per-distribution source - verification, lint-checking, etc. This acts as a minimal - gate, blocking extended use of VMs when a PR's code or commits - would otherwise not be accepted. Should run for less than a minute. +***N/B: Steps below are performed by automation*** -4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``. - This should execute within 10-minutes, but often much faster. +1. After `gating` passes, spin up one VM per + `matrix: image_name` item. Once accessible, ``ssh`` + into each VM as the `root` user. + +2. ``setup_environment.sh``: Configure root's `.bash_profile` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. 5. ``integration_test.sh``: Execute integration-testing. This is much more involved, and relies on access to external @@ -37,9 +48,12 @@ task (pass or fail) is set based on the exit status of the last script to execut Total execution time is capped at 2-hours (includes all the above) but this script normally completes in less than an hour. -### ``optional_system_testing`` Task -1. Optionally executes in parallel with ``full_vm_testing``. Requires +### ``optional_testing`` Task + +***N/B: Steps below are performed by automation*** + +1. Optionally executes in parallel with ``testing``. Requires **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` is found in the pull-request *description*. The *description* is the first text-box under the main *summary* line in the github WebUI. @@ -49,16 +63,17 @@ task (pass or fail) is set based on the exit status of the last script to execut 3. ``system_test.sh``: Build both dependencies and libpod, install them, then execute `make localsystem` from the repository root. -### ``build_vm_images`` Task -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), Cirrus - checks the last commit message. If it contains the magic string - ``***CIRRUS: REBUILD IMAGES***``, then this task continues. +### ``cache_images`` Task -2. Execute run another round of the ``full_vm_testing`` task (above). - After the tests pass (post-merge), spin up a special VM - (from the `image-builder-image`) capable of communicating with the - GCE API. Once accessible, ``ssh`` into the VM and run the following scripts. +***N/B: Steps below are performed by automation*** + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another + round of the ``gating`` and ``testing`` tasks (above). + +2. Assuming tests pass, if the commit message contains the magic string + ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise + simply mark the master branch as 'passed'. 3. ``setup_environment.sh``: Same as for other tasks. @@ -70,13 +85,108 @@ task (pass or fail) is set based on the exit status of the last script to execut 1. On a base-image VM, as root, copy the current state of the repository into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``full_vm_testing`` task (above). These scripts all - end with the suffix `_setup.sh` within the `$PACKER_BASE` directory. + use by the ``integration_testing`` task (above). For example, + ``fedora_setup.sh``. 3. If successful, shut down each VM and create a new GCE Image - named after the base image and the commit sha of the merge. + named with the base image, and the commit sha of the merge. ***Note:*** The ``.cirrus.yml`` file must be manually updated with the new images names, then the change sent in via a secondary pull-request. This -ensures that all the ``full_vm_testing`` tasks can pass with the new images, +ensures that all the ``integration_testing`` tasks can pass with the new images, before subjecting all future PRs to them. A workflow to automate this process is described in comments at the end of the ``.cirrus.yml`` file. + +### Base-images + +Base-images are VM disk-images specially prepared for executing as GCE VMs. +In particular, they run services on startup similar in purpose/function +as the standard 'cloud-init' services. + +* The google services are required for full support of ssh-key management + and GCE OAuth capabilities. Google provides native images in GCE + with services pre-installed, for many platforms. For example, + RHEL, CentOS, and Ubuntu. + +* Google does ***not*** provide any images for Fedora or Fedora Atomic + Host (as of 11/2018), nor do they provide a base-image prepared to + run packer for creating other images in the ``build_vm_images`` Task + (above). + +* Base images do not need to be produced often, but doing so completely + manually would be time-consuming and error-prone. Therefor a special + semi-automatic *Makefile* target is provided to assist with producing + all the base-images: ``libpod_base_images`` + +To produce new base-images, including an `image-builder-image` (used by +the ``cache_images`` Task) some input parameters are required: + + * ``GCP_PROJECT_ID``: The complete GCP project ID string e.g. foobar-12345 + identifying where the images will be stored. + + * ``GOOGLE_APPLICATION_CREDENTIALS``: A *JSON* file containing + credentials for a GCE service account. This can be [a service + account](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) + or [end-user + credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials] + + * ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths + to a `rhel-server-ec2-*.raw.xz` and it's cooresponding + checksum file. These must be supplied manually because + they're not available directly via URL like other images. + + * ``RHSM_COMMAND`` contains the complete string needed to register + the VM for installing package dependencies. The VM will be de-registered + upon completion. + + * Optionally, CSV's may be specified to ``PACKER_BUILDS`` + to limit the base-images produced. For example, + ``PACKER_BUILDS=fedora,image-builder-image``. + +The following process should be performed on a bare-metal CentOS 7 machine +with network access to GCE. Software dependencies can be obtained from +the ``packer/image-builder-image_base_setup.sh`` script. + +Alternatively, an existing image-builder-image may be used from within GCE. +However it must be created with elevated cloud privileges. For example, + +``` +$ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID + -e AS_USER=$USER -v /home/$USER:/home/$USER:z cevich/gcloud_centos:latest' + +$ URL=https://www.googleapis.com/auth +$ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control + +$ pgcloud compute instances create $USER-making-images \ + --image-family image-builder-image \ + --boot-disk-size "200GB" \ + --min-cpu-platform "Intel Haswell" \ + --machine-type n1-standard-2 \ + --scopes $SCOPES + +$ pgcloud compute ssh centos@$USER-making-images +... +``` + +When ready, change to the ``packer`` sub-directory, and run: + +``` +$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ + GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \ + RHEL_IMAGE_FILE=<VALUE> \ + RHEL_CSUM_FILE=<VALUE> \ + PACKER_BUILDS=<OPTIONAL> +``` + +Assuming this is successful (hence the semi-automatic part), packer will +produce a ``packer-manifest.json`` output file. This contains the base-image +names suitable for updating in ``.cirrus.yml``, `env` keys ``*_BASE_IMAGE``. + +On failure, it should be possible to determine the problem from the packer +output. The only exception is for the Fedora and FAH builds, which utilize +local qemu-kvm virtualisation. To observe the serial-port output from those +builds, set the ``TTYDEV`` parameter to your current device. For example: + +``` +$ make libpod_base_images ... TTYDEV=$(tty) + ... +``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index c8ff55445..ecdf1d877 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -8,12 +8,13 @@ CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT RUNC_COMMIT $RUNC_COMMIT PACKER_BUILDS $PACKER_BUILDS +BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE $CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE +FAH_BASE_IMAGE $FAH_BASE_IMAGE RHEL_BASE_IMAGE $RHEL_BASE_IMAGE RHSM_COMMAND $RHSM_COMMAND -BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX SERVICE_ACCOUNT $SERVICE_ACCOUNT GCE_SSH_USERNAME $GCE_SSH_USERNAME GCP_PROJECT_ID $GCP_PROJECT_ID @@ -28,28 +29,24 @@ show_env_vars # Assume basic dependencies are all met, but there could be a newer version # of the packer binary PACKER_FILENAME="packer_${PACKER_VER}_linux_amd64.zip" -mkdir -p "$HOME/packer" -cd "$HOME/packer" -# image_builder_image has packer pre-installed, check if same version requested -if ! [[ -r "$PACKER_FILENAME" ]] +if [[ -d "$HOME/packer" ]] then - curl -L -O https://releases.hashicorp.com/packer/$PACKER_VER/$PACKER_FILENAME - curl -L https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS | \ - grep 'linux_amd64' > ./sha256sums - sha256sum --check ./sha256sums - unzip -o $PACKER_FILENAME - ./packer --help &> /dev/null # verify exit(0) + cd "$HOME/packer" + # image_builder_image has packer pre-installed, check if same version requested + if [[ -r "$PACKER_FILENAME" ]] + then + cp $PACKER_FILENAME "$GOSRC/$PACKER_BASE/" + cp packer "$GOSRC/$PACKER_BASE/" + fi fi set -x -cd "$GOSRC" -# N/B: /usr/sbin/packer is a DIFFERENT tool, and will exit 0 given the args below :( -TEMPLATE="./$PACKER_BASE/libpod_images.json" - -$HOME/packer/packer inspect "$TEMPLATE" - -#$HOME/packer/packer build -machine-readable "-only=$PACKER_BUILDS" "$TEMPLATE" | tee /tmp/packer_log.csv -$HOME/packer/packer build "-only=$PACKER_BUILDS" "$TEMPLATE" - -# TODO: Report back to PR names of built images +cd "$GOSRC/$PACKER_BASE" +make libpod_images \ + PACKER_BUILDS=$PACKER_BUILDS \ + PACKER_VER=$PACKER_VER \ + GOSRC=$GOSRC \ + SCRIPT_BASE=$SCRIPT_BASE \ + PACKER_BASE=$PACKER_BASE \ + BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 226053724..a50bd448f 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" @@ -19,10 +19,13 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in make test-binaries "BUILDTAGS=$BUILDTAGS" SKIP_USERNS=1 make localintegration "BUILDTAGS=$BUILDTAGS" ;; - fedora-28) ;& # Continue to the next item + fedora-29) ;& # Continue to the next item + fedora-28) ;& centos-7) ;& rhel-7) - stub 'integration testing not working on $OS_RELEASE_ID' + make install PREFIX=/usr ETCDIR=/etc + make test-binaries + make localintegration ;; *) bad_os_id_ver ;; esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 04314e5fe..985264f22 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -4,8 +4,8 @@ # to be sourced by other scripts, not called directly. # Under some contexts these values are not set, make sure they are. -USER="$(whoami)" -HOME="$(getent passwd $USER | cut -d : -f 6)" +export USER="$(whoami)" +export HOME="$(getent passwd $USER | cut -d : -f 6)" if ! [[ "$PATH" =~ "/usr/local/bin" ]] then export PATH="$PATH:/usr/local/bin" @@ -73,6 +73,18 @@ PACKER_BUILDS $PACKER_BUILDS do [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\"" done + echo "" + echo "##### $(go version) #####" + echo "" +} + +# Unset environment variables not needed for testing purposes +clean_env() { + req_env_var " + UNSET_ENV_VARS $UNSET_ENV_VARS + " + echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables" + unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only } # Return a GCE image-name compatible string representation of distribution name @@ -269,21 +281,29 @@ install_varlink(){ } _finalize(){ + set +e # Don't fail at the very end + set +e # make errors non-fatal echo "Removing leftover giblets from cloud-init" cd / sudo rm -rf /var/lib/cloud/instance? sudo rm -rf /root/.ssh/* sudo rm -rf /home/* + sudo rm -rf /tmp/* + sudo rm -rf /tmp/.??* + sync + sudo fstrim -av } rh_finalize(){ + set +e # Don't fail at the very end # Allow root ssh-logins if [[ -r /etc/cloud/cloud.cfg ]] then sudo sed -re 's/^disable_root:.*/disable_root: 0/g' -i /etc/cloud/cloud.cfg fi echo "Resetting to fresh-state for usage as cloud-image." - sudo $(type -P dnf || type -P yum) clean all + PKG=$(type -P dnf || type -P yum || echo "") + [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic sudo rm -rf /var/cache/{yum,dnf} sudo rm -f /etc/udev/rules.d/*-persistent-*.rules sudo touch /.unconfigured # force firstboot to run @@ -291,7 +311,35 @@ rh_finalize(){ } ubuntu_finalize(){ + set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." sudo rm -rf /var/cache/apt _finalize } + +rhel_exit_handler() { + set +ex + req_env_var " + GOPATH $GOPATH + RHSMCMD $RHSMCMD + " + cd / + sudo rm -rf "$RHSMCMD" + sudo rm -rf "$GOPATH" + sudo subscription-manager remove --all + sudo subscription-manager unregister + sudo subscription-manager clean +} + +rhsm_enable() { + req_env_var " + RHSM_COMMAND $RHSM_COMMAND + " + export GOPATH="$(mktemp -d)" + export RHSMCMD="$(mktemp)" + trap "rhel_exit_handler" EXIT + # Avoid logging sensitive details + echo "$RHSM_COMMAND" > "$RHSMCMD" + ooe.sh sudo bash "$RHSMCMD" + sudo rm -rf "$RHSMCMD" +} diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore new file mode 100644 index 000000000..8f7bdeaf7 --- /dev/null +++ b/contrib/cirrus/packer/.gitignore @@ -0,0 +1,7 @@ +*json +packer +packer*zip +packer_cache +cidata* +meta-data +user-data diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile new file mode 100644 index 000000000..9bf27373e --- /dev/null +++ b/contrib/cirrus/packer/Makefile @@ -0,0 +1,108 @@ + +# N/B: PACKER_BUILDS variable is required. Should contain CSV of +# builder name(s) from applicable YAML file, +# e.g for names see libpod_images.yml + +PACKER_VER ?= 1.3.1 +PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip + +# Only needed for libpod_base_images target +TIMESTAMP := $(shell date +%s) +GOSRC ?= $(shell realpath "./../../../") +PACKER_BASE ?= contrib/cirrus/packer +SCRIPT_BASE ?= contrib/cirrus + +# For debugging nested-virt, use +#TTYDEV := $(shell tty) +TTYDEV := /dev/null + +.PHONY: all +all: libpod_images + +%.json: %.yml + @python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);' + +${PACKER_DIST_FILENAME}: + @curl -L --silent --show-error \ + -O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME} + +packer: ${PACKER_DIST_FILENAME} + @curl -L --silent --show-error \ + https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \ + | grep 'linux_amd64' > /tmp/packer_sha256sums + @sha256sum --check /tmp/packer_sha256sums + @unzip -o ${PACKER_DIST_FILENAME} + @touch --reference=Makefile ${PACKER_DIST_FILENAME} + +.PHONY: test +test: libpod_base_images.json libpod_images.json packer + ./packer inspect libpod_base_images.json > /dev/null + ./packer inspect libpod_images.json > /dev/null + @echo "All good" + +.PHONY: libpod_images +libpod_images: libpod_images.json packer +ifndef PACKER_BUILDS + $(error PACKER_BUILDS is undefined, expected builder-names CSV) +endif + ./packer build -only=${PACKER_BUILDS} \ + -var GOSRC=$(GOSRC) \ + -var PACKER_BASE=$(PACKER_BASE) \ + -var SCRIPT_BASE=$(SCRIPT_BASE) \ + libpod_images.json + @echo "" + @echo "Finished. The images mentioned above, and in packer-manifest.json" + @echo "can be used in .cirrus.yml as values for the 'image_name' keys" + @echo "" + +cidata.ssh: + ssh-keygen -f $@ -P "" -q + +cidata.ssh.pub: cidata.ssh + touch $@ + +meta-data: + echo "local-hostname: localhost.localdomain" > $@ + +user-data: cidata.ssh.pub + bash make-user-data.sh + +cidata.iso: user-data meta-data + genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data + +# This is intended to be run by a human, with admin access to the libpod GCE project. +.PHONY: libpod_base_images +libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer +ifndef GCP_PROJECT_ID + $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345) +endif +ifndef GOOGLE_APPLICATION_CREDENTIALS + $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) +endif +ifndef RHEL_IMAGE_FILE + $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) +endif +ifndef RHEL_CSUM_FILE + $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) +endif +ifndef RHSM_COMMAND + $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) +endif + PACKER_CACHE_DIR=/tmp ./packer build \ + -var TIMESTAMP=$(TIMESTAMP) \ + -var TTYDEV=$(TTYDEV) \ + -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ + -var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ + -var GOSRC=$(GOSRC) \ + -var PACKER_BASE=$(PACKER_BASE) \ + -var SCRIPT_BASE=$(SCRIPT_BASE) \ + -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.x86_64\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ + -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ + -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ + -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ + -only $(PACKER_BUILDS) \ + libpod_base_images.json + @echo "" + @echo "Finished. The images mentioned above, and in packer-manifest.json" + @echo "can be used in .cirrus.yml as values for the *_BASE_IMAGE keys." + @echo "" diff --git a/contrib/cirrus/packer/README.md b/contrib/cirrus/packer/README.md index 8ff6947e9..9a07ed960 100644 --- a/contrib/cirrus/packer/README.md +++ b/contrib/cirrus/packer/README.md @@ -1,2 +1,3 @@ These are definitions and scripts consumed by packer to produce the -various distribution images used for CI testing. +various distribution images used for CI testing. For more details +see the [Cirrus CI documentation](../README.md) diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh new file mode 100644 index 000000000..606c4f336 --- /dev/null +++ b/contrib/cirrus/packer/fah_base-setup.sh @@ -0,0 +1,45 @@ + +# N/B: This script is not intended to be run by humans. It is used to configure the +# FAH base image for importing, so that it will boot in GCE. + +set -e + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +install_ooe + +if [[ "$1" == "pre" ]] +then + echo "Upgrading Atomic Host" + setenforce 0 + ooe.sh atomic host upgrade + + echo "Configuring Repositories" + ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF +[ngompa-gce-oslogin] +name=Copr repo for gce-oslogin owned by ngompa +baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 +EOF + echo "Installing necessary packages and google services" + # Google services are enabled by default, upon install. + ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin + echo "Rebooting..." + systemctl reboot # Required for upgrade + package installs to be active +elif [[ "$1" == "post" ]] +then + echo "Enabling necessary services" + systemctl enable rngd # Must reboot before enabling + rh_finalize + echo "SUCCESS!" +else + echo "Expected to be called with 'pre' or 'post'" + exit 6 +fi diff --git a/contrib/cirrus/packer/fah_setup.sh b/contrib/cirrus/packer/fah_setup.sh new file mode 100644 index 000000000..2e053b396 --- /dev/null +++ b/contrib/cirrus/packer/fah_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script is called by packer on the subject fah VM, to setup the podman +# build/test environment. It's not intended to be used outside of this context. + +set -e + +# Load in library (copied by packer, before this script was run) +source /tmp/libpod/$SCRIPT_BASE/lib.sh + +req_env_var " +SCRIPT_BASE $SCRIPT_BASE +" + +install_ooe + +ooe.sh sudo atomic host upgrade + +ooe.sh sudo rpm-ostree uninstall cloud-init + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh new file mode 100644 index 000000000..c0a1e422c --- /dev/null +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# N/B: This script is not intended to be run by humans. It is used to configure the +# fedora base image for importing, so that it will boot in GCE + +set -e + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +[[ "$1" == "post" ]] || exit 0 # nothing to do + +install_ooe + +echo "Updating packages" +ooe.sh dnf -y update + +echo "Installing necessary packages and google services" +ooe.sh dnf -y copr enable ngompa/gce-oslogin +ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin + +echo "Enabling services" +ooe.sh systemctl enable rngd + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index f9fea04a7..4e4391e59 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -10,6 +10,7 @@ source /tmp/libpod/$SCRIPT_BASE/lib.sh req_env_var " SCRIPT_BASE $SCRIPT_BASE +FEDORA_CNI_COMMIT $FEDORA_CNI_COMMIT CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT CRIU_COMMIT $CRIU_COMMIT @@ -65,11 +66,13 @@ ooe.sh sudo dnf install -y \ runc \ skopeo-containers \ slirp4netns \ + unzip \ which \ xz install_varlink +CNI_COMMIT=$FEDORA_CNI_COMMIT install_cni_plugins install_buildah diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh new file mode 100644 index 000000000..b8e2824a7 --- /dev/null +++ b/contrib/cirrus/packer/image-builder-image_base-setup.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# This script is called by packer on a vanilla CentOS VM, to setup the image +# used for building images FROM base images. It's not intended to be used +# outside of this context. + +set -e + +[[ "$1" == "post" ]] || exit 0 # pre stage not needed + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +req_env_var " + TIMESTAMP $TIMESTAMP + GOSRC $GOSRC + SCRIPT_BASE $SCRIPT_BASE + PACKER_BASE $PACKER_BASE +" + +install_ooe + +echo "Updating packages" +ooe.sh sudo yum -y update + +echo "Configuring repositories" +ooe.sh sudo yum -y install centos-release-scl epel-release + +echo "Installing packages" +ooe.sh sudo yum -y install \ + genisoimage \ + golang \ + google-cloud-sdk \ + libvirt \ + libvirt-admin \ + libvirt-client \ + libvirt-daemon \ + make \ + python34 \ + python34 \ + python34-PyYAML \ + python34-PyYAML \ + qemu-img \ + qemu-kvm \ + qemu-kvm-tools \ + qemu-user \ + rsync \ + unzip \ + util-linux \ + vim + +sudo ln -s /usr/libexec/qemu-kvm /usr/bin/ + +sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF +options kvm-intel nested=1 +options kvm-intel enable_shadow_vmcs=1 +options kvm-intel enable_apicv=1 +options kvm-intel ept=1 +EOF + +echo "Installing packer" +sudo mkdir -p /root/$(basename $PACKER_BASE) +sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE) +sudo mkdir -p /root/$(basename $SCRIPT_BASE) +sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE) + +install_scl_git + +echo "Cleaning up" +cd / +rm -rf $GOSRC + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml new file mode 100644 index 000000000..4ae44e0d9 --- /dev/null +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -0,0 +1,179 @@ +--- + +variables: + # Complete local path to this repository (Required) + GOSRC: + # Relative path to this (packer) subdirectory (Required) + PACKER_BASE: + # Relative path to cirrus scripts subdirectory (Required) + SCRIPT_BASE: + # Unique ID for naming new base-images (required) + TIMESTAMP: + # Required for output from qemu builders + TTYDEV: + # RHEL images require click-through agreements to obtain (required) + RHEL_BASE_IMAGE_NAME: + RHEL_IMAGE_FILE: + RHEL_CSUM_FILE: + # RHEL requires a subscription to install/update packages + RHSM_COMMAND: + + # Fedora images are obtainable by direct download + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE + FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2" + FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM" + FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE + + # The name of the image in GCE used for packer build libpod_images.yml + IBI_BASE_NAME: 'image-builder-image' + CIDATA_ISO: 'cidata.iso' # produced by Makefile + + # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json) + GOOGLE_APPLICATION_CREDENTIALS: + # The complete project ID (required, not the short name) + GCP_PROJECT_ID: + # Pre-existing storage bucket w/ lifecycle-enabled + XFERBUCKET: "packer-import" # pre-created, globally unique, lifecycle-enabled + +# Don't leak sensitive values in error messages / output +sensitive-variables: + - 'GOOGLE_APPLICATION_CREDENTIALS' + - 'GCP_PROJECT_ID' + - 'RHSM_COMMAND' + +# What images to produce in which cloud +builders: + - name: '{{user `IBI_BASE_NAME`}}' + type: 'googlecompute' + image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}' + image_family: '{{user `IBI_BASE_NAME`}}' + source_image_project_id: 'centos-cloud' + source_image_family: 'centos-7' + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + communicator: 'ssh' + ssh_username: 'centos' + ssh_pty: 'true' + # The only supported zone in Cirrus-CI, as of addition of this comment + zone: 'us-central1-a' + # Enable nested virtualization in case it's ever needed + image_licenses: + - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx' + min_cpu_platform: "Intel Broadwell" # nested-virt requirement + + - &nested_virt + name: 'fedora' + type: 'qemu' + accelerator: "kvm" + iso_url: '{{user `FEDORA_IMAGE_URL`}}' + disk_image: true + format: "raw" + disk_size: 5120 + iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}' + iso_checksum_type: "sha256" + output_directory: '/tmp/{{build_name}}' + vm_name: "disk.raw" # actually qcow2, name required for post-processing + boot_wait: '5s' + shutdown_command: 'shutdown -h now' + headless: true + qemu_binary: "/usr/libexec/qemu-kvm" + qemuargs: # List-of-list format required to override packer-generated args + - - "-m" + - "1024" + - - "-cpu" + - "host" + - - "-device" + - "virtio-rng-pci" + - - "-chardev" + - "tty,id=pts,path={{user `TTYDEV`}}" + - - "-device" + - "isa-serial,chardev=pts" + - - "-cdrom" + - "{{user `CIDATA_ISO`}}" + - - "-netdev" + - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22" + - - "-device" + - "virtio-net,netdev=net0" + communicator: 'ssh' + ssh_private_key_file: 'cidata.ssh' + ssh_username: 'root' + + - <<: *nested_virt + name: 'fah' + iso_url: '{{user `FAH_IMAGE_URL`}}' + iso_checksum_url: '{{user `FAH_CSUM_URL`}}' + disk_size: 10240 + + - <<: *nested_virt + name: 'rhel' + iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}' + iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}' + disk_size: 10240 + +provisioners: + - type: 'shell' + inline: + - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' + - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' + + - type: 'file' + source: '{{user `GOSRC`}}/.cirrus.yml' + destination: '/tmp/libpod/.cirrus.yml' + + - type: 'file' + source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' + destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' + + - type: 'file' + source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' + destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' + + - &shell_script + type: 'shell' + inline: + - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre' + expect_disconnect: true # Allow this to reboot the VM + environment_vars: + - 'TIMESTAMP={{user `TIMESTAMP`}}' + - 'GOSRC=/tmp/libpod' + - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' + - 'PACKER_BASE={{user `PACKER_BASE`}}' + - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' + + - <<: *shell_script + inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] + expect_disconnect: false + pause_before: '10s' + inline: + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post' + +post-processors: + - - type: "compress" + only: ['fedora', 'fah', 'rhel'] + output: '/tmp/{{build_name}}/disk.raw.tar.gz' + format: '.tar.gz' + compression_level: 9 + - &gcp_import + only: ['fedora'] + type: "googlecompute-import" + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + bucket: '{{user `XFERBUCKET`}}' + gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}-{{uuid}}.tar.gz' + image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' + image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' + - <<: *gcp_import + only: ['fah'] + image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `FAH_IMAGE_URL`}}' + image_family: '{{user `FAH_BASE_IMAGE_NAME`}}' + - <<: *gcp_import + only: ['rhel'] + image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}' + image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}' + - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json deleted file mode 100644 index 9dac3e8ea..000000000 --- a/contrib/cirrus/packer/libpod_images.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "variables": { - "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}", - "CNI_COMMIT": "{{env `CNI_COMMIT`}}", - "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}", - "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}", - "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}", - - "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" , - "UBUNTU_BASE_IMAGE": "{{env `UBUNTU_BASE_IMAGE`}}", - "FEDORA_BASE_IMAGE": "{{env `FEDORA_BASE_IMAGE`}}", - "RHEL_BASE_IMAGE": "{{env `RHEL_BASE_IMAGE`}}", - - "GOSRC": "{{env `GOSRC`}}", - "PACKER_BASE": "{{env `PACKER_BASE`}}", - "SCRIPT_BASE": "{{env `SCRIPT_BASE`}}", - - "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}", - "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}", - "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}", - "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}", - "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}" - }, - "sensitive-variables": [ - "GCP_PROJECT_ID", "SERVICE_ACCOUNT", "GCE_SSH_USERNAME", "RHSM_COMMAND" - ], - "builders": [ - { - "name": "rhel-7", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `RHEL_BASE_IMAGE`}}", - "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "ec2-user", - "ssh_pty": "true" - },{ - "name": "centos-7", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `CENTOS_BASE_IMAGE`}}", - "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "{{user `GCE_SSH_USERNAME`}}", - "ssh_pty": "true" - },{ - "name": "fedora-28", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `FEDORA_BASE_IMAGE`}}", - "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "fedora", - "ssh_pty": "true" - },{ - "name": "ubuntu-18", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `UBUNTU_BASE_IMAGE`}}", - "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "{{user `GCE_SSH_USERNAME`}}", - "ssh_pty": "true" - } - ], - "provisioners": [ - { - "type": "file", - "source": "{{user `GOSRC`}}", - "destination": "/tmp/libpod" - },{ - "type": "shell", - "only": ["rhel-7"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/rhel_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}", - "RHSM_COMMAND={{user `RHSM_COMMAND`}}" - ] - },{ - "type": "shell", - "only": ["centos-7"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/centos_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - },{ - "type": "shell", - "only": ["fedora-28"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - },{ - "type": "shell", - "only": ["ubuntu-18"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/ubuntu_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - } - ] -} diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml new file mode 100644 index 000000000..7b95b08cc --- /dev/null +++ b/contrib/cirrus/packer/libpod_images.yml @@ -0,0 +1,91 @@ +--- + +# All of these are required +variables: + # Names of GCE Base images to start from, in .cirrus.yml + RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}' + CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}' + UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' + FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' + FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}' + + # libpod dependencies to build and install into images + FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}" + CNI_COMMIT: "{{env `CNI_COMMIT`}}" + CRIO_COMMIT: "{{env `CRIO_COMMIT`}}" + CRIU_COMMIT: "{{env `CRIU_COMMIT`}}" + RUNC_COMMIT: "{{env `RUNC_COMMIT`}}" + + BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' + GOSRC: '{{env `GOSRC`}}' + PACKER_BASE: '{{env `PACKER_BASE`}}' + SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' + + # Protected credentials, decrypted by Cirrus at runtime + GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' + GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' + RHSM_COMMAND: '{{env `RHSM_COMMAND`}}' + SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' + GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' + +# Don't leak sensitive values in error messages / output +sensitive-variables: + - 'GCE_SSH_USERNAME' + - 'GCP_PROJECT_ID' + - 'RHSM_COMMAND' + - 'SERVICE_ACCOUNT' + +# What images to produce in which cloud +builders: + # v----- is a YAML anchor, allows referencing this object by name (below) + - &gce_hosted_image + name: 'ubuntu-18' + type: 'googlecompute' + image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' + image_family: '{{build_name}}-libpod' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + disk_size: 20 + project_id: '{{user `GCP_PROJECT_ID`}}' + service_account_email: '{{user `SERVICE_ACCOUNT`}}' + communicator: 'ssh' + ssh_username: '{{user `GCE_SSH_USERNAME`}}' + ssh_pty: 'true' + # The only supported zone in Cirrus-CI, as of addition of this comment + zone: 'us-central1-a' + + # v----- is a YAML alias, allows partial re-use of the anchor object + - <<: *gce_hosted_image + name: 'rhel-7' + source_image: '{{user `RHEL_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'centos-7' + source_image: '{{user `CENTOS_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'fedora-29' + source_image: '{{user `FEDORA_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'fah-29' + source_image: '{{user `FAH_BASE_IMAGE`}}' + +# The brains of the operation, making actual modifications to the base-image. +provisioners: + - type: 'file' + source: '{{user `GOSRC`}}' + destination: '/tmp/libpod' + + - type: 'shell' + script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh' + environment_vars: + - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' + - 'CNI_COMMIT={{user `CNI_COMMIT`}}' + - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}' + - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}' + - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}' + - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}' + - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' + +post-processors: + - - type: 'manifest' diff --git a/contrib/cirrus/packer/make-user-data.sh b/contrib/cirrus/packer/make-user-data.sh new file mode 100644 index 000000000..7f7fa1c1a --- /dev/null +++ b/contrib/cirrus/packer/make-user-data.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script is utilized by Makefile, it's not intended to be run by humans + +cat <<EOF > user-data +#cloud-config +timezone: US/Eastern +growpart: + mode: auto +disable_root: false +ssh_pwauth: True +ssh_import_id: [root] +ssh_authorized_keys: + - $(cat cidata.ssh.pub) +users: + - name: root + primary-group: root + homedir: /root + system: true +EOF diff --git a/contrib/cirrus/packer/rhel_base-setup.sh b/contrib/cirrus/packer/rhel_base-setup.sh new file mode 100644 index 000000000..8b2073d4f --- /dev/null +++ b/contrib/cirrus/packer/rhel_base-setup.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# N/B: This script is not intended to be run by humans. It is used to configure the +# rhel base image for importing, so that it will boot in GCE + +set -e + +[[ "$1" == "post" ]] || exit 0 # pre stage is not needed + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +req_env_var " + RHSM_COMMAND $RHSM_COMMAND +" + +install_ooe + +echo "Setting up repos" +# Frequently needed +ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + +# Required for google to manage ssh keys +ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM +[google-cloud-compute] +name=google-cloud-compute +baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOM + +rhsm_enable + +echo "Installing/removing packages" +ooe.sh sudo yum -y install google-compute-engine google-compute-engine-oslogin +ooe.sh sudo yum -y erase "cloud-init" "rh-amazon-rhui-client*" || true +ooe.sh sudo systemctl enable \ + google-accounts-daemon \ + google-clock-skew-daemon \ + google-instance-setup \ + google-network-daemon \ + google-shutdown-scripts \ + google-startup-scripts + +rhel_exit_handler # release subscription! + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index d296713fc..7f0d4e589 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -18,24 +18,7 @@ RHSM_COMMAND $RHSM_COMMAND install_ooe -export GOPATH="$(mktemp -d)" -export RHSMCMD="$(mktemp)" - -exit_handler() { - set +ex - cd / - sudo rm -rf "$RHSMCMD" - sudo rm -rf "$GOPATH" - sudo subscription-manager remove --all - sudo subscription-manager unregister - sudo subscription-manager clean -} -trap "exit_handler" EXIT - -# Avoid logging sensitive details -echo "$RHSM_COMMAND" > "$RHSMCMD" -ooe.sh sudo bash "$RHSMCMD" -sudo rm -rf "$RHSMCMD" +rhsm_enable ooe.sh sudo yum -y erase "rh-amazon-rhui-client*" ooe.sh sudo subscription-manager repos "--disable=*" @@ -47,21 +30,6 @@ ooe.sh sudo subscription-manager repos \ ooe.sh sudo yum -y update -# Frequently needed -ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - -# Required for google to manage ssh keys -sudo tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM -[google-cloud-compute] -name=google-cloud-compute -baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOM - ooe.sh sudo yum -y install \ atomic-registries \ btrfs-progs-devel \ @@ -74,8 +42,6 @@ ooe.sh sudo yum -y install \ golang \ golang-github-cpuguy83-go-md2man \ golang-github-cpuguy83-go-md2man \ - google-compute-engine \ - google-compute-engine-oslogin \ gpgme-devel \ iptables \ libassuan-devel \ @@ -118,7 +84,7 @@ install_criu install_packer_copied_files -exit_handler # release subscription! +rhel_exit_handler # release subscription! rh_finalize diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index ef209a4a4..6e3613462 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -21,6 +21,7 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT +# Avoid getting stuck waiting for user input export DEBIAN_FRONTEND=noninteractive # Try twice as workaround for minor networking problems @@ -56,6 +57,8 @@ ooe.sh sudo -E apt-get -qq install --no-install-recommends \ libostree-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ + libseccomp-dev \ + libseccomp2 \ libtool \ libudev-dev \ lsof \ diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2563b5f43..5ba842cf1 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -16,12 +16,11 @@ CIRRUS_BUILD_ID $CIRRUS_BUILD_ID" cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions # Verify basic dependencies -for depbin in go rsync unzip sha256sum curl make +for depbin in go rsync unzip sha256sum curl make python3 git do if ! type -P "$depbin" &> /dev/null then - echo "ERROR: $depbin binary not found in $PATH" - exit 2 + echo "***** WARNING: $depbin binary not found in $PATH *****" fi done @@ -35,14 +34,15 @@ then # N/B: Single-quote items evaluated every time, double-quotes only once (right now). for envstr in \ "$MARK" \ + "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \ "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \ "export TRAVIS=\"1\"" \ "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \ "export OS_RELEASE_ID=\"$(os_release_id)\"" \ "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ - "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \ + "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \ "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \ - "export GOPATH=\"/go\"" \ + "export GOPATH=\"/var/tmp/go\"" \ 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' do @@ -57,7 +57,8 @@ then install_runc_from_git envstr='export BUILDTAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) varlink exclude_graphdriver_devicemapper"' ;; - fedora-28) ;& # Continue to the next item + fedora-29) ;& # Continue to the next item + fedora-28) ;& centos-7) ;& rhel-7) envstr='unset BUILDTAGS' # Use default from Makefile diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index 7c727d336..66974f8c6 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index cacc23045..e5b167e79 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -9,22 +9,22 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in ubuntu-18) + make install.tools "BUILDTAGS=$BUILDTAGS" make localunit "BUILDTAGS=$BUILDTAGS" make "BUILDTAGS=$BUILDTAGS" ;; - fedora-28) + fedora-29) ;& # Continue to the next item + centos-7) ;& + rhel-7) + make install.tools make localunit make ;; - centos-7) ;& # Continue to the next item - rhel-7) - stub 'unit testing not working on $OS_RELEASE_ID' - ;; *) bad_os_id_ver ;; esac diff --git a/contrib/cirrus/verify_source.sh b/contrib/cirrus/verify_source.sh deleted file mode 100755 index 860bafc00..000000000 --- a/contrib/cirrus/verify_source.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e -source $(dirname $0)/lib.sh - -req_env_var " -OS_RELEASE_ID $OS_RELEASE_ID -OS_RELEASE_VER $OS_RELEASE_VER -" - -show_env_vars - -set -x -cd "$GOSRC" - -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) - make install.tools "BUILDTAGS=$BUILDTAGS" - make validate "BUILDTAGS=$BUILDTAGS" - # make lint "BUILDTAGS=$BUILDTAGS" - ;; - fedora-28) ;& - centos-7) ;& - rhel-7) - make install.tools - make validate - # make lint - ;; - *) bad_os_id_ver ;; -esac diff --git a/contrib/python/podman/Makefile b/contrib/python/podman/Makefile index 11a7568d1..0cbfe2fb3 100644 --- a/contrib/python/podman/Makefile +++ b/contrib/python/podman/Makefile @@ -1,6 +1,6 @@ PYTHON ?= $(shell command -v python3 2>/dev/null || command -v python) DESTDIR ?= / -PODMAN_VERSION ?= '0.0.4' +PODMAN_VERSION ?= '0.11.1.1' .PHONY: python-podman python-podman: @@ -22,8 +22,8 @@ install: .PHONY: upload upload: - $(PODMAN_VERSION) $(PYTHON) setup.py sdist bdist_wheel - twine upload --repository-url https://test.pypi.org/legacy/ dist/* + PODMAN_VERSION=$(PODMAN_VERSION) $(PYTHON) setup.py sdist bdist_wheel + twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/* .PHONY: clobber clobber: uninstall clean diff --git a/contrib/python/pypodman/Makefile b/contrib/python/pypodman/Makefile index 272145c5d..230eee44d 100644 --- a/contrib/python/pypodman/Makefile +++ b/contrib/python/pypodman/Makefile @@ -1,6 +1,6 @@ PYTHON ?= $(shell command -v python3 2>/dev/null || command -v python) DESTDIR := / -PODMAN_VERSION ?= '0.0.4' +PODMAN_VERSION ?= '0.11.1.1' .PHONY: python-pypodman python-pypodman: @@ -22,7 +22,7 @@ install: .PHONY: upload upload: - $(PODMAN_VERSION) $(PYTHON) setup.py sdist bdist_wheel + PODMAN_VERSION=$(PODMAN_VERSION) $(PYTHON) setup.py sdist bdist_wheel twine upload --repository-url https://test.pypi.org/legacy/ dist/* .PHONY: clobber diff --git a/contrib/python/pypodman/pypodman/lib/__init__.py b/contrib/python/pypodman/pypodman/lib/__init__.py index be1b5f467..d9a434254 100644 --- a/contrib/python/pypodman/pypodman/lib/__init__.py +++ b/contrib/python/pypodman/pypodman/lib/__init__.py @@ -3,18 +3,17 @@ import sys import podman from pypodman.lib.action_base import AbstractActionBase -from pypodman.lib.parser_actions import (BooleanAction, BooleanValidate, - ChangeAction, PathAction, - PositiveIntAction, UnitAction) +from pypodman.lib.parser_actions import (ChangeAction, PathAction, + PositiveIntAction, SignalAction, + UnitAction) from pypodman.lib.podman_parser import PodmanArgumentParser from pypodman.lib.report import Report, ReportColumn # Silence pylint overlording... -assert BooleanAction -assert BooleanValidate assert ChangeAction assert PathAction assert PositiveIntAction +assert SignalAction assert UnitAction __all__ = [ diff --git a/contrib/python/pypodman/pypodman/lib/action_base.py b/contrib/python/pypodman/pypodman/lib/action_base.py index a950c362b..5cba7ac5c 100644 --- a/contrib/python/pypodman/pypodman/lib/action_base.py +++ b/contrib/python/pypodman/pypodman/lib/action_base.py @@ -17,29 +17,21 @@ class AbstractActionBase(abc.ABC): Use set_defaults() to set attributes "class_" and "method". These will be invoked as class_(parsed_args).method() """ - parent.add_argument( + parent.add_flag( '--all', - action='store_true', - help=('list all items.' - ' (default: no-op, included for compatibility.)')) - parent.add_argument( - '--no-trunc', - '--notruncate', - action='store_false', - dest='truncate', + help='list all items.') + parent.add_flag( + '--truncate', + '--trunc', default=True, - help='Display extended information. (default: False)') - parent.add_argument( - '--noheading', - action='store_false', - dest='heading', + help="Truncate id's and other long fields.") + parent.add_flag( + '--heading', default=True, - help=('Omit the table headings from the output.' - ' (default: False)')) - parent.add_argument( + help='Include table headings in the output.') + parent.add_flag( '--quiet', - action='store_true', - help='List only the IDs. (default: %(default)s)') + help='List only the IDs.') def __init__(self, args): """Construct class.""" diff --git a/contrib/python/pypodman/pypodman/lib/actions/_create_args.py b/contrib/python/pypodman/pypodman/lib/actions/_create_args.py index 207f52796..8ab4292e8 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/_create_args.py +++ b/contrib/python/pypodman/pypodman/lib/actions/_create_args.py @@ -1,6 +1,6 @@ """Implement common create container arguments together.""" -from pypodman.lib import BooleanAction, UnitAction +from pypodman.lib import SignalAction, UnitAction class CreateArguments(): @@ -108,11 +108,9 @@ class CreateArguments(): metavar='NODES', help=('Memory nodes (MEMs) in which to allow execution (0-3, 0,1).' ' Only effective on NUMA systems')) - parser.add_argument( + parser.add_flag( '--detach', '-d', - action=BooleanAction, - default=False, help='Detached mode: run the container in the background and' ' print the new container ID. (default: False)') parser.add_argument( @@ -218,7 +216,7 @@ class CreateArguments(): # only way for argparse to handle these options. vol_args = { - 'choices': ['bind', 'tmpfs', 'ignore'], + 'choices': ('bind', 'tmpfs', 'ignore'), 'metavar': 'MODE', 'type': str.lower, 'help': 'Tells podman how to handle the builtin image volumes', @@ -228,12 +226,10 @@ class CreateArguments(): volume_group.add_argument('--image-volume', **vol_args) volume_group.add_argument('--builtin-volume', **vol_args) - parser.add_argument( + parser.add_flag( '--interactive', '-i', - action=BooleanAction, - default=False, - help='Keep STDIN open even if not attached. (default: False)') + help='Keep STDIN open even if not attached.') parser.add_argument('--ipc', help='Create namespace') parser.add_argument( '--kernel-memory', action=UnitAction, help='Kernel memory limit') @@ -278,10 +274,9 @@ class CreateArguments(): metavar='BRIDGE', help='Set the Network mode for the container.' ' (format: bridge, host, container:UUID, ns:PATH, none)') - parser.add_argument( + parser.add_flag( '--oom-kill-disable', - action=BooleanAction, - help='Whether to disable OOM Killer for the container or not') + help='Whether to disable OOM Killer for the container or not.') parser.add_argument( '--oom-score-adj', choices=range(-1000, 1000), @@ -298,41 +293,33 @@ class CreateArguments(): help=("Tune the container's pids limit." " Set -1 to have unlimited pids for the container.")) parser.add_argument('--pod', help='Run container in an existing pod') - parser.add_argument( + parser.add_flag( '--privileged', - action=BooleanAction, help='Give extended privileges to this container.') parser.add_argument( '--publish', '-p', metavar='RANGE', help="Publish a container's port, or range of ports, to the host") - parser.add_argument( + parser.add_flag( '--publish-all', '-P', - action=BooleanAction, help='Publish all exposed ports to random' - ' ports on the host interfaces' - '(default: False)') - parser.add_argument( + ' ports on the host interfaces.') + parser.add_flag( '--quiet', '-q', - action='store_true', help='Suppress output information when pulling images') - parser.add_argument( + parser.add_flag( '--read-only', - action=BooleanAction, help="Mount the container's root filesystem as read only.") - parser.add_argument( + parser.add_flag( '--rm', - action=BooleanAction, - default=False, help='Automatically remove the container when it exits.') parser.add_argument( '--rootfs', - action='store_true', - help=('If specified, the first argument refers to an' - ' exploded container on the file system of remote host.')) + help='If specified, the first argument refers to an' + ' exploded container on the file system of remote host.') parser.add_argument( '--security-opt', action='append', @@ -340,15 +327,14 @@ class CreateArguments(): help='Set security options.') parser.add_argument( '--shm-size', action=UnitAction, help='Size of /dev/shm') - parser.add_argument( + parser.add_flag( '--sig-proxy', - action=BooleanAction, - default=True, help='Proxy signals sent to the podman run' ' command to the container process') parser.add_argument( '--stop-signal', - metavar='SIGTERM', + action=SignalAction, + default='TERM', help='Signal to stop a container') parser.add_argument( '--stop-timeout', @@ -374,11 +360,9 @@ class CreateArguments(): metavar='MOUNT', help='Create a tmpfs mount.' ' (default: rw,noexec,nosuid,nodev,size=65536k.)') - parser.add_argument( + parser.add_flag( '--tty', '-t', - action=BooleanAction, - default=False, help='Allocate a pseudo-TTY for standard input of container.') parser.add_argument( '--uidmap', @@ -394,15 +378,16 @@ class CreateArguments(): parser.add_argument( '--user', '-u', - help=('Sets the username or UID used and optionally' - ' the groupname or GID for the specified command.')) + help='Sets the username or UID used and optionally' + ' the groupname or GID for the specified command.') parser.add_argument( '--userns', metavar='NAMESPACE', help='Set the user namespace mode for the container') parser.add_argument( '--uts', - choices=['host', 'ns'], + choices=('host', 'ns'), + type=str.lower, help='Set the UTS mode for the container') parser.add_argument('--volume', '-v', help='Create a bind mount.') parser.add_argument( diff --git a/contrib/python/pypodman/pypodman/lib/actions/commit_action.py b/contrib/python/pypodman/pypodman/lib/actions/commit_action.py index 21924e938..c166e1aff 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/commit_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/commit_action.py @@ -2,7 +2,7 @@ import sys import podman -from pypodman.lib import AbstractActionBase, BooleanAction, ChangeAction +from pypodman.lib import AbstractActionBase, ChangeAction class Commit(AbstractActionBase): @@ -44,17 +44,14 @@ class Commit(AbstractActionBase): help='Set commit message for committed image' ' (Only on docker images.)', ) - parser.add_argument( + parser.add_flag( '--pause', '-p', - action=BooleanAction, - default=True, help='Pause the container when creating an image', ) - parser.add_argument( + parser.add_flag( '--quiet', '-q', - action='store_true', help='Suppress output', ) parser.add_argument( diff --git a/contrib/python/pypodman/pypodman/lib/actions/history_action.py b/contrib/python/pypodman/pypodman/lib/actions/history_action.py index f9aaa54f6..76c3ad756 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/history_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/history_action.py @@ -5,8 +5,7 @@ from collections import OrderedDict import humanize import podman -from pypodman.lib import (AbstractActionBase, BooleanAction, Report, - ReportColumn) +from pypodman.lib import AbstractActionBase, Report, ReportColumn class History(AbstractActionBase): @@ -17,13 +16,10 @@ class History(AbstractActionBase): """Add History command to parent parser.""" parser = parent.add_parser('history', help='report image history') super().subparser(parser) - parser.add_argument( + parser.add_flag( '--human', '-H', - action=BooleanAction, - default='True', - help='Display sizes and dates in human readable format.' - ' (default: %(default)s)') + help='Display sizes and dates in human readable format.') parser.add_argument( '--format', choices=('json', 'table'), diff --git a/contrib/python/pypodman/pypodman/lib/actions/images_action.py b/contrib/python/pypodman/pypodman/lib/actions/images_action.py index 29bf90dd2..21376eeeb 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/images_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/images_action.py @@ -24,11 +24,9 @@ class Images(AbstractActionBase): help=('Change sort ordered of displayed images.' ' (default: %(default)s)')) - group = parser.add_mutually_exclusive_group() - group.add_argument( + parser.add_flag( '--digests', - action='store_true', - help='Include digests with images. (default: %(default)s)') + help='Include digests with images.') parser.set_defaults(class_=cls, method='list') def __init__(self, args): diff --git a/contrib/python/pypodman/pypodman/lib/actions/info_action.py b/contrib/python/pypodman/pypodman/lib/actions/info_action.py index 988284541..3c854a358 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/info_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/info_action.py @@ -22,10 +22,6 @@ class Info(AbstractActionBase): " (default: yaml)") parser.set_defaults(class_=cls, method='info') - def __init__(self, args): - """Construct Info class.""" - super().__init__(args) - def info(self): """Report on Podman Service.""" try: diff --git a/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py b/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py index a581e7e4e..ca5ad2215 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py @@ -22,12 +22,9 @@ class Inspect(AbstractActionBase): type=str.lower, help='Type of object to inspect', ) - parser.add_argument( + parser.add_flag( '--size', - action='store_true', - default=False, - help='Display the total file size if the type is a container.' - ' Always True.') + help='Display the total file size if the type is a container.') parser.add_argument( 'objects', nargs='+', @@ -35,10 +32,6 @@ class Inspect(AbstractActionBase): ) parser.set_defaults(class_=cls, method='inspect') - def __init__(self, args): - """Construct Inspect class.""" - super().__init__(args) - def _get_container(self, ident): try: logging.debug("Getting container %s", ident) diff --git a/contrib/python/pypodman/pypodman/lib/actions/kill_action.py b/contrib/python/pypodman/pypodman/lib/actions/kill_action.py index cb3d3f035..e8fb4e74d 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/kill_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/kill_action.py @@ -1,9 +1,8 @@ """Remote client command for signaling podman containers.""" -import signal import sys import podman -from pypodman.lib import AbstractActionBase +from pypodman.lib import AbstractActionBase, SignalAction class Kill(AbstractActionBase): @@ -16,10 +15,9 @@ class Kill(AbstractActionBase): parser.add_argument( '--signal', '-s', - choices=range(1, signal.NSIG), - metavar='[1,{}]'.format(signal.NSIG), + action=SignalAction, default=9, - help='Signal to send to the container. (default: 9)') + help='Signal to send to the container. (default: %(default)s)') parser.add_argument( 'containers', nargs='+', @@ -27,10 +25,6 @@ class Kill(AbstractActionBase): ) parser.set_defaults(class_=cls, method='kill') - def __init__(self, args): - """Construct Kill class.""" - super().__init__(args) - def kill(self): """Signal provided containers.""" try: diff --git a/contrib/python/pypodman/pypodman/lib/actions/pause_action.py b/contrib/python/pypodman/pypodman/lib/actions/pause_action.py index ab64d8b81..7dc02f7fe 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pause_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pause_action.py @@ -19,10 +19,6 @@ class Pause(AbstractActionBase): ) parser.set_defaults(class_=cls, method='pause') - def __init__(self, args): - """Construct Pause class.""" - super().__init__(args) - def pause(self): """Pause provided containers.""" try: diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/create_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/create_parser.py index 46c1e3e51..4e0bde777 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/create_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/create_parser.py @@ -2,7 +2,7 @@ import sys import podman -from pypodman.lib import AbstractActionBase, BooleanAction +from pypodman.lib import AbstractActionBase class CreatePod(AbstractActionBase): @@ -20,12 +20,9 @@ class CreatePod(AbstractActionBase): type=str, help='Path to cgroups under which the' ' cgroup for the pod will be created.') - parser.add_argument( + parser.add_flag( '--infra', - action=BooleanAction, - default=True, - help='Create an infra container and associate it with the pod' - '(default: %(default)s)') + help='Create an infra container and associate it with the pod.') parser.add_argument( '-l', '--label', diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/kill_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/kill_parser.py index 430ec34e0..9b6229939 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/kill_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/kill_parser.py @@ -3,7 +3,7 @@ import signal import sys import podman -from pypodman.lib import AbstractActionBase +from pypodman.lib import AbstractActionBase, SignalAction from pypodman.lib import query_model as query_pods @@ -15,18 +15,16 @@ class KillPod(AbstractActionBase): """Add Pod Kill command to parent parser.""" parser = parent.add_parser('kill', help='signal containers in pod') - parser.add_argument( - '-a', + parser.add_flag( '--all', - action='store_true', - help='Sends signal to all pods') + '-a', + help='Sends signal to all pods.') parser.add_argument( '-s', '--signal', - choices=range(1, signal.NSIG), - metavar='[1,{}]'.format(signal.NSIG), + action=SignalAction, default=9, - help='Signal to send to the pod. (default: 9)') + help='Signal to send to the pod. (default: %(default)s)') parser.add_argument('pod', nargs='*', help='pod(s) to signal') parser.set_defaults(class_=cls, method='kill') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/pause_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/pause_parser.py index daae028d4..c751314ca 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/pause_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/pause_parser.py @@ -13,8 +13,10 @@ class PausePod(AbstractActionBase): def subparser(cls, parent): """Add Pod Pause command to parent parser.""" parser = parent.add_parser('pause', help='pause containers in pod') - parser.add_argument( - '-a', '--all', action='store_true', help='Pause all pods') + parser.add_flag( + '--all', + '-a', + help='Pause all pods.') parser.add_argument('pod', nargs='*', help='pod(s) to pause.') parser.set_defaults(class_=cls, method='pause') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/processes_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/processes_parser.py index ecfcb883a..855e313c7 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/processes_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/processes_parser.py @@ -14,18 +14,15 @@ class ProcessesPod(AbstractActionBase): parser = parent.add_parser('ps', help='list processes of pod') super().subparser(parser) - parser.add_argument( + parser.add_flag( '--ctr-names', - action='store_true', - help='Include container name in the info field') - parser.add_argument( + help='Include container name in the info field.') + parser.add_flag( '--ctr-ids', - action='store_true', - help='Include container ID in the info field') - parser.add_argument( + help='Include container ID in the info field.') + parser.add_flag( '--ctr-status', - action='store_true', - help='Include container status in the info field') + help='Include container status in the info field.') parser.add_argument( '--format', choices=('json'), diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/remove_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/remove_parser.py index 40eeb7203..289325d14 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/remove_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/remove_parser.py @@ -13,13 +13,14 @@ class RemovePod(AbstractActionBase): def subparser(cls, parent): """Add Pod Rm command to parent parser.""" parser = parent.add_parser('rm', help='Delete pod and container(s)') - parser.add_argument( - '-a', '--all', action='store_true', help='Remove all pods') - parser.add_argument( - '-f', + parser.add_flag( + '--all', + '-a', + help='Remove all pods.') + parser.add_flag( '--force', - action='store_true', - help='Stop and remove container(s) then delete pod') + '-f', + help='Stop and remove container(s) then delete pod.') parser.add_argument( 'pod', nargs='*', help='Pod to remove. Or, use --all') parser.set_defaults(class_=cls, method='remove') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/restart_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/restart_parser.py index af489ad28..53f45b6de 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/restart_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/restart_parser.py @@ -13,8 +13,10 @@ class RestartPod(AbstractActionBase): def subparser(cls, parent): """Add Pod Restart command to parent parser.""" parser = parent.add_parser('restart', help='restart containers in pod') - parser.add_argument( - '-a', '--all', action='store_true', help='Restart all pods') + parser.add_flag( + '--all', + '-a', + help='Restart all pods.') parser.add_argument( 'pod', nargs='*', help='Pod to restart. Or, use --all') parser.set_defaults(class_=cls, method='restart') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/start_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/start_parser.py index 0ddc336bf..ff62b839e 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/start_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/start_parser.py @@ -14,8 +14,10 @@ class StartPod(AbstractActionBase): def subparser(cls, parent): """Add Pod Start command to parent parser.""" parser = parent.add_parser('start', help='start pod') - parser.add_argument( - '-a', '--all', action='store_true', help='Start all pods') + parser.add_flag( + '--all', + '-a', + help='Start all pods.') parser.add_argument( 'pod', nargs='*', help='Pod to start. Or, use --all') parser.set_defaults(class_=cls, method='start') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/stop_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/stop_parser.py index 7054fd38a..cbf2bf1e7 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/stop_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/stop_parser.py @@ -13,8 +13,10 @@ class StopPod(AbstractActionBase): def subparser(cls, parent): """Add Pod Stop command to parent parser.""" parser = parent.add_parser('stop', help='stop pod') - parser.add_argument( - '-a', '--all', action='store_true', help='Stop all pods') + parser.add_flag( + '--all', + '-a', + help='Stop all pods.') parser.add_argument( 'pod', nargs='*', help='Pod to stop. Or, use --all') parser.set_defaults(class_=cls, method='stop') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod/unpause_parser.py b/contrib/python/pypodman/pypodman/lib/actions/pod/unpause_parser.py index 90e1ddbe2..5186cf9cc 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod/unpause_parser.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod/unpause_parser.py @@ -13,8 +13,10 @@ class UnpausePod(AbstractActionBase): def subparser(cls, parent): """Add Pod Unpause command to parent parser.""" parser = parent.add_parser('unpause', help='unpause pod') - parser.add_argument( - '-a', '--all', action='store_true', help='Unpause all pods') + parser.add_flag( + '--all', + '-a', + help='Unpause all pods.') parser.add_argument( 'pod', nargs='*', help='Pod to unpause. Or, use --all') parser.set_defaults(class_=cls, method='unpause') diff --git a/contrib/python/pypodman/pypodman/lib/actions/pod_action.py b/contrib/python/pypodman/pypodman/lib/actions/pod_action.py index 046af34bb..4b8997a05 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/pod_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/pod_action.py @@ -5,6 +5,8 @@ import sys from pypodman.lib import AbstractActionBase +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import from .pod import * diff --git a/contrib/python/pypodman/pypodman/lib/actions/port_action.py b/contrib/python/pypodman/pypodman/lib/actions/port_action.py index d2a8ded46..6913f3813 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/port_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/port_action.py @@ -13,16 +13,13 @@ class Port(AbstractActionBase): """Add Port command to parent parser.""" parser = parent.add_parser( 'port', help='retrieve ports from containers') - parser.add_argument( + parser.add_flag( '--all', '-a', - action='store_true', - default=False, help='List all known port mappings for running containers') parser.add_argument( 'containers', nargs='*', - default=None, help='containers to list ports', ) parser.set_defaults(class_=cls, method='port') @@ -61,3 +58,4 @@ class Port(AbstractActionBase): file=sys.stderr, flush=True) return 1 + return 0 diff --git a/contrib/python/pypodman/pypodman/lib/actions/push_action.py b/contrib/python/pypodman/pypodman/lib/actions/push_action.py index 0030cb5b9..8e86ca335 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/push_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/push_action.py @@ -15,12 +15,10 @@ class Push(AbstractActionBase): 'push', help='push image elsewhere', ) - parser.add_argument( + parser.add_flag( '--tlsverify', - action='store_true', - default=True, help='Require HTTPS and verify certificates when' - ' contacting registries (default: %(default)s)') + ' contacting registries.') parser.add_argument( 'image', nargs=1, help='name or id of image to push') parser.add_argument( @@ -30,10 +28,6 @@ class Push(AbstractActionBase): ) parser.set_defaults(class_=cls, method='push') - def __init__(self, args): - """Construct Push class.""" - super().__init__(args) - def pull(self): """Store image elsewhere.""" try: diff --git a/contrib/python/pypodman/pypodman/lib/actions/restart_action.py b/contrib/python/pypodman/pypodman/lib/actions/restart_action.py index d99d1ad65..415594920 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/restart_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/restart_action.py @@ -23,10 +23,6 @@ class Restart(AbstractActionBase): 'targets', nargs='+', help='container id(s) to restart') parser.set_defaults(class_=cls, method='restart') - def __init__(self, args): - """Construct Restart class.""" - super().__init__(args) - def restart(self): """Restart container(s).""" try: diff --git a/contrib/python/pypodman/pypodman/lib/actions/rm_action.py b/contrib/python/pypodman/pypodman/lib/actions/rm_action.py index e8074ef4e..99ff6c460 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/rm_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/rm_action.py @@ -12,20 +12,14 @@ class Rm(AbstractActionBase): def subparser(cls, parent): """Add Rm command to parent parser.""" parser = parent.add_parser('rm', help='delete container(s)') - parser.add_argument( - '-f', + parser.add_flag( '--force', - action='store_true', - help=('force delete of running container(s).' - ' (default: %(default)s)')) + '-f', + help='force delete of running container(s).') parser.add_argument( 'targets', nargs='+', help='container id(s) to delete') parser.set_defaults(class_=cls, method='remove') - def __init__(self, args): - """Construct Rm class.""" - super().__init__(args) - def remove(self): """Remove container(s).""" for ident in self._args.targets: diff --git a/contrib/python/pypodman/pypodman/lib/actions/rmi_action.py b/contrib/python/pypodman/pypodman/lib/actions/rmi_action.py index c6ba835cb..7c3d0bd79 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/rmi_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/rmi_action.py @@ -12,19 +12,13 @@ class Rmi(AbstractActionBase): def subparser(cls, parent): """Add Rmi command to parent parser.""" parser = parent.add_parser('rmi', help='delete image(s)') - parser.add_argument( - '-f', + parser.add_flag( '--force', - action='store_true', - help=('force delete of image(s) and associated containers.' - ' (default: %(default)s)')) + '-f', + help='force delete of image(s) and associated containers.') parser.add_argument('targets', nargs='+', help='image id(s) to delete') parser.set_defaults(class_=cls, method='remove') - def __init__(self, args): - """Construct Rmi class.""" - super().__init__(args) - def remove(self): """Remove image(s).""" for ident in self._args.targets: diff --git a/contrib/python/pypodman/pypodman/lib/actions/search_action.py b/contrib/python/pypodman/pypodman/lib/actions/search_action.py index d2a585d92..b7b8b465d 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/search_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/search_action.py @@ -4,8 +4,8 @@ import sys from collections import OrderedDict import podman -from pypodman.lib import (AbstractActionBase, BooleanValidate, - PositiveIntAction, Report, ReportColumn) +from pypodman.lib import (AbstractActionBase, PositiveIntAction, Report, + ReportColumn) class FilterAction(argparse.Action): @@ -58,16 +58,16 @@ class FilterAction(argparse.Action): if val < 0: parser.error(msg) elif opt == 'is-automated': - try: - val = BooleanValidate()(val) - except ValueError: + if val.capitalize() in ('True', 'False'): + val = bool(val) + else: msg = ('{} option "is-automated"' ' must be True or False.'.format(self.dest)) parser.error(msg) elif opt == 'is-official': - try: - val = BooleanValidate()(val) - except ValueError: + if val.capitalize() in ('True', 'False'): + val = bool(val) + else: msg = ('{} option "is-official"' ' must be True or False.'.format(self.dest)) parser.error(msg) diff --git a/contrib/python/pypodman/pypodman/lib/actions/start_action.py b/contrib/python/pypodman/pypodman/lib/actions/start_action.py index f312fb3fa..5f88731dc 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/start_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/start_action.py @@ -2,7 +2,7 @@ import sys import podman -from pypodman.lib import AbstractActionBase, BooleanAction +from pypodman.lib import AbstractActionBase class Start(AbstractActionBase): @@ -12,12 +12,10 @@ class Start(AbstractActionBase): def subparser(cls, parent): """Add Start command to parent parser.""" parser = parent.add_parser('start', help='start container') - parser.add_argument( + parser.add_flag( '--attach', '-a', - action=BooleanAction, - default=False, - help="Attach container's STDOUT and STDERR (default: %(default)s)") + help="Attach container's STDOUT and STDERR.") parser.add_argument( '--detach-keys', metavar='KEY(s)', @@ -25,18 +23,14 @@ class Start(AbstractActionBase): help='Override the key sequence for detaching a container.' ' (format: a single character [a-Z] or ctrl-<value> where' ' <value> is one of: a-z, @, ^, [, , or _) (default: ^D)') - parser.add_argument( + parser.add_flag( '--interactive', '-i', - action=BooleanAction, - default=False, - help="Attach container's STDIN (default: %(default)s)") + help="Attach container's STDIN.") # TODO: Implement sig-proxy - parser.add_argument( + parser.add_flag( '--sig-proxy', - action=BooleanAction, - default=False, - help="Proxy received signals to the process (default: %(default)s)" + help="Proxy received signals to the process." ) parser.add_argument( 'containers', @@ -74,3 +68,4 @@ class Start(AbstractActionBase): file=sys.stderr, flush=True) return 1 + return 0 diff --git a/contrib/python/pypodman/pypodman/lib/actions/version_action.py b/contrib/python/pypodman/pypodman/lib/actions/version_action.py index 12b6dc576..29a0cabe4 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/version_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/version_action.py @@ -1,9 +1,7 @@ """Remote client command for reporting on Podman service.""" -import json import sys import podman -import yaml from pypodman.lib import AbstractActionBase @@ -17,10 +15,6 @@ class Version(AbstractActionBase): 'version', help='report version on podman service') parser.set_defaults(class_=cls, method='version') - def __init__(self, args): - """Construct Version class.""" - super().__init__(args) - def version(self): """Report on Podman Service.""" try: diff --git a/contrib/python/pypodman/pypodman/lib/parser_actions.py b/contrib/python/pypodman/pypodman/lib/parser_actions.py index 77ee14761..3ff12cab8 100644 --- a/contrib/python/pypodman/pypodman/lib/parser_actions.py +++ b/contrib/python/pypodman/pypodman/lib/parser_actions.py @@ -6,6 +6,7 @@ The constructors are very verbose but remain for IDE support. import argparse import copy import os +import signal # API defined by argparse.Action therefore shut up pylint # pragma pylint: disable=redefined-builtin @@ -13,22 +14,8 @@ import os # pragma pylint: disable=too-many-arguments -class BooleanValidate(): - """Validate value is boolean string.""" - - def __call__(self, value): - """Return True, False or raise ValueError.""" - val = value.capitalize() - if val == 'False': - return False - elif val == 'True': - return True - else: - raise ValueError('"{}" is not True or False'.format(value)) - - -class BooleanAction(argparse.Action): - """Convert and validate bool argument.""" +class ChangeAction(argparse.Action): + """Convert and validate change argument.""" def __init__(self, option_strings, @@ -40,8 +27,13 @@ class BooleanAction(argparse.Action): choices=None, required=False, help=None, - metavar='{True,False}'): - """Create BooleanAction object.""" + metavar='OPT=VALUE'): + """Create ChangeAction object.""" + help = (help or '') + ('Apply change(s) to the new image.' + ' May be given multiple times.') + if default is None: + default = [] + super().__init__( option_strings=option_strings, dest=dest, @@ -56,32 +48,37 @@ class BooleanAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): """Convert and Validate input.""" - try: - val = BooleanValidate()(values) - except ValueError: - parser.error('"{}" must be True or False.'.format(option_string)) - else: - setattr(namespace, self.dest, val) + items = getattr(namespace, self.dest, None) or [] + items = copy.copy(items) + choices = ('CMD', 'ENTRYPOINT', 'ENV', 'EXPOSE', 'LABEL', 'ONBUILD', + 'STOPSIGNAL', 'USER', 'VOLUME', 'WORKDIR') + + opt, _ = values.split('=', 1) + if opt not in choices: + parser.error('Option "{}" is not supported by argument "{}",' + ' valid options are: {}'.format( + opt, option_string, ', '.join(choices))) + items.append(values) + setattr(namespace, self.dest, items) -class ChangeAction(argparse.Action): - """Convert and validate change argument.""" + +class SignalAction(argparse.Action): + """Validate input as a signal.""" def __init__(self, option_strings, dest, nargs=None, const=None, - default=[], - type=None, + default=None, + type=str, choices=None, required=False, - help=None, - metavar='OPT=VALUE'): - """Create ChangeAction object.""" - help = (help or '') + ('Apply change(s) to the new image.' - ' May be given multiple times.') - + help='The signal to send.' + ' It may be given as a name or a number.', + metavar='SIGNAL'): + """Create SignalAction object.""" super().__init__( option_strings=option_strings, dest=dest, @@ -94,21 +91,40 @@ class ChangeAction(argparse.Action): help=help, metavar=metavar) - def __call__(self, parser, namespace, values, option_string=None): - """Convert and Validate input.""" - items = getattr(namespace, self.dest, None) or [] - items = copy.copy(items) + if hasattr(signal, "Signals"): - choices = ('CMD', 'ENTRYPOINT', 'ENV', 'EXPOSE', 'LABEL', 'ONBUILD', - 'STOPSIGNAL', 'USER', 'VOLUME', 'WORKDIR') + def _signal_number(signame): + cooked = 'SIG{}'.format(signame) + try: + return signal.Signals[cooked].value + except ValueError: + pass + else: - opt, val = values.split('=', 1) - if opt not in choices: - parser.error('Option "{}" is not supported by argument "{}",' - ' valid options are: {}'.format( - opt, option_string, ', '.join(choices))) - items.append(values) - setattr(namespace, self.dest, items) + def _signal_number(signame): + cooked = 'SIG{}'.format(signame) + for n, v in sorted(signal.__dict__.items()): + if n != cooked: + continue + if n.startswith("SIG") and not n.startswith("SIG_"): + return v + + self._signal_number = _signal_number + + def __call__(self, parser, namespace, values, option_string=None): + """Validate input is a signal for platform.""" + if values.isdigit(): + signum = int(values) + if signal.SIGRTMIN <= signum >= signal.SIGRTMAX: + raise ValueError('"{}" is not a valid signal. {}-{}'.format( + values, signal.SIGRTMIN, signal.SIGRTMAX)) + else: + signum = self._signal_number(values) + if signum is None: + parser.error( + '"{}" is not a valid signal,' + ' see your platform documentation.'.format(values)) + setattr(namespace, self.dest, signum) class UnitAction(argparse.Action): diff --git a/contrib/python/pypodman/pypodman/lib/podman_parser.py b/contrib/python/pypodman/pypodman/lib/podman_parser.py index 412c8c8fd..913546a91 100644 --- a/contrib/python/pypodman/pypodman/lib/podman_parser.py +++ b/contrib/python/pypodman/pypodman/lib/podman_parser.py @@ -48,6 +48,18 @@ class PodmanArgumentParser(argparse.ArgumentParser): super().__init__(**kwargs) + def add_flag(self, *args, **kwargs): + """Add flag to parser.""" + flags = [a for a in args if a[0] in self.prefix_chars] + dest = flags[0].lstrip(self.prefix_chars) + no_flag = '{0}{0}no-{1}'.format(self.prefix_chars, dest) + + group = self.add_mutually_exclusive_group(required=False) + group.add_argument(*flags, action='store_true', dest=dest, **kwargs) + group.add_argument(no_flag, action='store_false', dest=dest, **kwargs) + default = kwargs.get('default', False) + self.set_defaults(**{dest: default}) + def initialize_parser(self): """Initialize parser without causing recursion meltdown.""" self.add_argument( diff --git a/docs/podman-container-runlabel.1.md b/docs/podman-container-runlabel.1.md index 73b7d7e15..6f7b4dae8 100644 --- a/docs/podman-container-runlabel.1.md +++ b/docs/podman-container-runlabel.1.md @@ -95,8 +95,8 @@ option be used, as the default behavior of using the system-wide default policy **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used, If set to false then tls verification will not be used. If not specified -tls verification will be used unless the target registry is listed as an insecure registry in registries.conf +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf ## Examples ## diff --git a/docs/podman-generate-kube.1.md b/docs/podman-generate-kube.1.md new file mode 100644 index 000000000..59c3353a5 --- /dev/null +++ b/docs/podman-generate-kube.1.md @@ -0,0 +1,119 @@ +% podman-generate Podman Man Pages +% Brent Baude +% December 2018 +# NAME +podman-generate-kube - Generate Kubernetes YAML + +# SYNOPSIS +**podman generate kube ** +[**-h**|**--help**] +[**-s**][**--service**] +CONTAINER|POD + +# DESCRIPTION +**podman generate kube** will generate Kubernetes Pod YAML (v1 specification) from a podman container or pod. Whether +the input is for a container or pod, Podman will always generate the specification as a Pod. The input may be in the form +of a pod or container name or ID. + +The **service** option can be used to generate a Service specification for the corresponding Pod ouput. In particular, +if the object has portmap bindings, the service specification will include a NodePort declaration to expose the service. A +random port is assigned by Podman in the specification. + +# OPTIONS: + +**s** **--service** +Generate a service file for the resulting Pod YAML. + +## Examples ## + +Create Kubernetes Pod YAML for a container called `some-mariadb` . +``` +$ sudo podman generate kube some-mariadb +# Generation of Kubenetes YAML is still under development! +# +# Save the output of this file and use kubectl create -f to import +# it into Kubernetes. +# +# Created with podman-0.11.2-dev +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: 2018-12-03T19:07:59Z + labels: + app: some-mariadb + name: some-mariadb-libpod +spec: + containers: + - command: + - docker-entrypoint.sh + - mysqld + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: TERM + value: xterm + - name: HOSTNAME + - name: container + value: podman + - name: GOSU_VERSION + value: "1.10" + - name: GPG_KEYS + value: "199369E5404BD5FC7D2FE43BCBCB082A1BB943DB \t177F4010FE56CA3336300305F1656F24C74CD1D8 + \t430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A \t4D1BB29D63D98E422B2113B19334A25F8507EFA5" + - name: MARIADB_MAJOR + value: "10.3" + - name: MARIADB_VERSION + value: 1:10.3.10+maria~bionic + - name: MYSQL_ROOT_PASSWORD + value: x + image: quay.io/baude/demodb:latest + name: some-mariadb + ports: + - containerPort: 3306 + hostPort: 36533 + protocol: TCP + resources: {} + securityContext: + allowPrivilegeEscalation: true + privileged: false + readOnlyRootFilesystem: false + tty: true + workingDir: / +status: {} +``` + +Create Kubernetes service YAML for a container called `some-mariabdb` +``` +$ sudo podman generate kube -s some-mariadb +# Generation of Kubenetes YAML is still under development! +# +# Save the output of this file and use kubectl create -f to import +# it into Kubernetes. +# +# Created with podman-0.11.2-dev +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: 2018-12-03T19:08:24Z + labels: + app: some-mariadb + name: some-mariadb-libpod +spec: + ports: + - name: "3306" + nodePort: 30929 + port: 3306 + protocol: TCP + targetPort: 0 + selector: + app: some-mariadb + type: NodePort +status: + loadBalancer: {} +``` + +## SEE ALSO +podman(1), podman-container, podman-pod + +# HISTORY +Decemeber 2018, Originally compiled by Brent Baude (bbaude at redhat dot com) diff --git a/docs/podman-generate.1.md b/docs/podman-generate.1.md new file mode 100644 index 000000000..f19f48511 --- /dev/null +++ b/docs/podman-generate.1.md @@ -0,0 +1,19 @@ +% podman-generate(1) + +## NAME +podman\-container - generate structured data based for a containers and pods + +## SYNOPSIS +**podman generate** *subcommand* + +## DESCRIPTION +The generate command will create structured output (like YAML) based on a container or pod. + +## COMMANDS + +| Command | Man Page | Description | +| ------- | --------------------------------------------------- | ---------------------------------------------------------------------------- | +| kube | [podman-generate-kube(1)](podman-generate-kube.1.md) | Generate Kubernetes YAML based on a pod or container + +## SEE ALSO +podman, podman-pod, podman-container diff --git a/docs/podman-login.1.md b/docs/podman-login.1.md index a3ee2929c..7c033d7c5 100644 --- a/docs/podman-login.1.md +++ b/docs/podman-login.1.md @@ -43,7 +43,9 @@ Default certificates directory is _/etc/containers/certs.d_. **--tls-verify** -Require HTTPS and verify certificates when contacting registries (default: true) +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. **--help**, **-h** diff --git a/docs/podman-pod-stop.1.md b/docs/podman-pod-stop.1.md index 74799273e..7544f8bf7 100644 --- a/docs/podman-pod-stop.1.md +++ b/docs/podman-pod-stop.1.md @@ -19,26 +19,48 @@ Stops all pods Instead of providing the pod name or ID, stop the last created pod. +**--timeout, --time, t** + +Timeout to wait before forcibly stopping the containers in the pod. + ## EXAMPLE -podman pod stop mywebserverpod +Stop a pod called *mywebserverpod* +``` +$ podman pod stop mywebserverpod cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 +``` -podman pod stop 490eb 3557fb +Stop two pods by their short IDs. +``` +$ podman pod stop 490eb 3557fb 490eb241aaf704d4dd2629904410fe4aa31965d9310a735f8755267f4ded1de5 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab +``` +Stop the most recent pod +``` +$ podman pod stop --latest 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab +``` -podman pod stop --latest -3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab - -podman pod stop --all +Stop all pods +``` +$ podman pod stop --all 19456b4cd557eaf9629825113a552681a6013f8c8cad258e36ab825ef536e818 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab 490eb241aaf704d4dd2629904410fe4aa31965d9310a735f8755267f4ded1de5 70c358daecf71ef9be8f62404f926080ca0133277ef7ce4f6aa2d5af6bb2d3e9 cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 +``` + +Stop all pods with a timeout of 1 second. +``` +$ podman pod stop -a -t 1 +3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab +490eb241aaf704d4dd2629904410fe4aa31965d9310a735f8755267f4ded1de5 +70c358daecf71ef9be8f62404f926080ca0133277ef7ce4f6aa2d5af6bb2d3e9 +``` ## SEE ALSO podman-pod(1), podman-pod-start(1), podman-stop(1) diff --git a/docs/podman-pull.1.md b/docs/podman-pull.1.md index 86c6823af..2196e251e 100644 --- a/docs/podman-pull.1.md +++ b/docs/podman-pull.1.md @@ -77,8 +77,8 @@ option be used, as the default behavior of using the system-wide default policy **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used, If set to false then tls verification will not be used. If not specified -tls verification will be used unless the target registry is listed as an insecure registry in registries.conf. +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. **--help**, **-h** diff --git a/docs/podman-push.1.md b/docs/podman-push.1.md index 537988ea0..3ce156010 100644 --- a/docs/podman-push.1.md +++ b/docs/podman-push.1.md @@ -93,7 +93,9 @@ Add a signature at the destination using the specified key **--tls-verify** -Require HTTPS and verify certificates when contacting registries (default: true) +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. ## EXAMPLE diff --git a/docs/podman-search.1.md b/docs/podman-search.1.md index ea1228f94..61f50f1dc 100644 --- a/docs/podman-search.1.md +++ b/docs/podman-search.1.md @@ -72,8 +72,8 @@ Do not truncate the output **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used. If set to false then tls verification will not be used if needed. If not specified -default registries will be searched through (in /etc/containers/registries.conf), and tls will be skipped if a default +then TLS verification will be used. If set to false, then TLS verification will not be used if needed. If not specified, +default registries will be searched through (in /etc/containers/registries.conf), and TLS will be skipped if a default registry is listed in the insecure registries. **--help**, **-h** diff --git a/libpod/common/common.go b/libpod/common/common.go index 932f1f6da..5d10bee36 100644 --- a/libpod/common/common.go +++ b/libpod/common/common.go @@ -1,32 +1,9 @@ package common import ( - "io" - - cp "github.com/containers/image/copy" "github.com/containers/image/types" ) -// GetCopyOptions constructs a new containers/image/copy.Options{} struct from the given parameters -func GetCopyOptions(reportWriter io.Writer, signaturePolicyPath string, srcDockerRegistry, destDockerRegistry *DockerRegistryOptions, signing SigningOptions, authFile, manifestType string, forceCompress bool) *cp.Options { - if srcDockerRegistry == nil { - srcDockerRegistry = &DockerRegistryOptions{} - } - if destDockerRegistry == nil { - destDockerRegistry = &DockerRegistryOptions{} - } - srcContext := srcDockerRegistry.GetSystemContext(signaturePolicyPath, authFile, forceCompress) - destContext := destDockerRegistry.GetSystemContext(signaturePolicyPath, authFile, forceCompress) - return &cp.Options{ - RemoveSignatures: signing.RemoveSignatures, - SignBy: signing.SignBy, - ReportWriter: reportWriter, - SourceCtx: srcContext, - DestinationCtx: destContext, - ForceManifestMIMEType: manifestType, - } -} - // GetSystemContext Constructs a new containers/image/types.SystemContext{} struct from the given signaturePolicy path func GetSystemContext(signaturePolicyPath, authFilePath string, forceCompress bool) *types.SystemContext { sc := &types.SystemContext{} diff --git a/libpod/common/docker_registry_options.go b/libpod/common/docker_registry_options.go deleted file mode 100644 index f79ae0c54..000000000 --- a/libpod/common/docker_registry_options.go +++ /dev/null @@ -1,35 +0,0 @@ -package common - -import "github.com/containers/image/types" - -// DockerRegistryOptions encapsulates settings that affect how we connect or -// authenticate to a remote registry. -type DockerRegistryOptions struct { - // DockerRegistryCreds is the user name and password to supply in case - // we need to pull an image from a registry, and it requires us to - // authenticate. - DockerRegistryCreds *types.DockerAuthConfig - // DockerCertPath is the location of a directory containing CA - // certificates which will be used to verify the registry's certificate - // (all files with names ending in ".crt"), and possibly client - // certificates and private keys (pairs of files with the same name, - // except for ".cert" and ".key" suffixes). - DockerCertPath string - // DockerInsecureSkipTLSVerify turns off verification of TLS - // certificates and allows connecting to registries without encryption. - DockerInsecureSkipTLSVerify bool -} - -// GetSystemContext constructs a new system context from the given signaturePolicy path and the -// values in the DockerRegistryOptions -func (o DockerRegistryOptions) GetSystemContext(signaturePolicyPath, authFile string, forceCompress bool) *types.SystemContext { - sc := &types.SystemContext{ - SignaturePolicyPath: signaturePolicyPath, - DockerAuthConfig: o.DockerRegistryCreds, - DockerCertPath: o.DockerCertPath, - DockerInsecureSkipTLSVerify: o.DockerInsecureSkipTLSVerify, - AuthFilePath: authFile, - DirForceCompress: forceCompress, - } - return sc -} diff --git a/libpod/common/output_interfaces.go b/libpod/common/output_interfaces.go deleted file mode 100644 index 805d0c79a..000000000 --- a/libpod/common/output_interfaces.go +++ /dev/null @@ -1 +0,0 @@ -package common diff --git a/libpod/container_internal_unsupported.go b/libpod/container_internal_unsupported.go index eed0449a9..4af0cd56c 100644 --- a/libpod/container_internal_unsupported.go +++ b/libpod/container_internal_unsupported.go @@ -28,10 +28,10 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { return nil, ErrNotImplemented } -func (c *Container) checkpoint(ctx context.Context, keep bool) error { +func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointOptions) error { return ErrNotImplemented } -func (c *Container) restore(ctx context.Context, keep bool) error { +func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) error { return ErrNotImplemented } diff --git a/libpod/image/docker_registry_options.go b/libpod/image/docker_registry_options.go index 97a151396..c191a3ca2 100644 --- a/libpod/image/docker_registry_options.go +++ b/libpod/image/docker_registry_options.go @@ -19,8 +19,9 @@ type DockerRegistryOptions struct { // except for ".cert" and ".key" suffixes). DockerCertPath string // DockerInsecureSkipTLSVerify turns off verification of TLS - // certificates and allows connecting to registries without encryption. - DockerInsecureSkipTLSVerify bool + // certificates and allows connecting to registries without encryption + // - or forces it on even if registries.conf has the registry configured as insecure. + DockerInsecureSkipTLSVerify types.OptionalBool } // GetSystemContext constructs a new system context from a parent context. the values in the DockerRegistryOptions, and other parameters. diff --git a/libpod/image/image.go b/libpod/image/image.go index 434f9031e..476d28226 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -125,7 +125,7 @@ func (ir *Runtime) NewFromLocal(name string) (*Image, error) { // New creates a new image object where the image could be local // or remote -func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile string, writer io.Writer, dockeroptions *DockerRegistryOptions, signingoptions SigningOptions, forcePull, forceSecure bool) (*Image, error) { +func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile string, writer io.Writer, dockeroptions *DockerRegistryOptions, signingoptions SigningOptions, forcePull bool) (*Image, error) { // We don't know if the image is local or not ... check local first newImage := Image{ InputName: name, @@ -145,7 +145,7 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, forceSecure) + imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", name) } @@ -167,7 +167,7 @@ func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.Im if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, false) + imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef)) } @@ -498,7 +498,7 @@ func (i *Image) UntagImage(tag string) error { // PushImageToHeuristicDestination pushes the given image to "destination", which is heuristically parsed. // Use PushImageToReference if the destination is known precisely. -func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, forceSecure bool, additionalDockerArchiveTags []reference.NamedTagged) error { +func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, additionalDockerArchiveTags []reference.NamedTagged) error { if destination == "" { return errors.Wrapf(syscall.EINVAL, "destination image name must be specified") } @@ -516,11 +516,11 @@ func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination return err } } - return i.PushImageToReference(ctx, dest, manifestMIMEType, authFile, signaturePolicyPath, writer, forceCompress, signingOptions, dockerRegistryOptions, forceSecure, additionalDockerArchiveTags) + return i.PushImageToReference(ctx, dest, manifestMIMEType, authFile, signaturePolicyPath, writer, forceCompress, signingOptions, dockerRegistryOptions, additionalDockerArchiveTags) } // PushImageToReference pushes the given image to a location described by the given path -func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageReference, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, forceSecure bool, additionalDockerArchiveTags []reference.NamedTagged) error { +func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageReference, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, additionalDockerArchiveTags []reference.NamedTagged) error { sc := GetSystemContext(signaturePolicyPath, authFile, forceCompress) policyContext, err := getPolicyContext(sc) @@ -534,23 +534,8 @@ func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageRefere if err != nil { return errors.Wrapf(err, "error getting source imageReference for %q", i.InputName) } - insecureRegistries, err := registries.GetInsecureRegistries() - if err != nil { - return err - } copyOptions := getCopyOptions(sc, writer, nil, dockerRegistryOptions, signingOptions, manifestMIMEType, additionalDockerArchiveTags) - if dest.Transport().Name() == DockerTransport { - imgRef := dest.DockerReference() - if imgRef == nil { // This should never happen; such references can’t be created. - return fmt.Errorf("internal error: DockerTransport reference %s does not have a DockerReference", transports.ImageName(dest)) - } - registry := reference.Domain(imgRef) - - if util.StringInSlice(registry, insecureRegistries) && !forceSecure { - copyOptions.DestinationCtx.DockerInsecureSkipTLSVerify = true - logrus.Info(fmt.Sprintf("%s is an insecure registry; pushing with tls-verify=false", registry)) - } - } + copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath() // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. // Copy the image to the remote destination _, err = cp.Image(ctx, policyContext, dest, src, copyOptions) if err != nil { diff --git a/libpod/image/image_test.go b/libpod/image/image_test.go index f187631b4..91bb2411b 100644 --- a/libpod/image/image_test.go +++ b/libpod/image/image_test.go @@ -86,9 +86,9 @@ func TestImage_NewFromLocal(t *testing.T) { // Need images to be present for this test ir, err := NewImageRuntimeFromOptions(so) assert.NoError(t, err) - bb, err := ir.New(context.Background(), "docker.io/library/busybox:latest", "", "", writer, nil, SigningOptions{}, false, false) + bb, err := ir.New(context.Background(), "docker.io/library/busybox:latest", "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) - bbglibc, err := ir.New(context.Background(), "docker.io/library/busybox:glibc", "", "", writer, nil, SigningOptions{}, false, false) + bbglibc, err := ir.New(context.Background(), "docker.io/library/busybox:glibc", "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) tm, err := makeLocalMatrix(bb, bbglibc) @@ -135,7 +135,7 @@ func TestImage_New(t *testing.T) { // Iterate over the names and delete the image // after the pull for _, img := range names { - newImage, err := ir.New(context.Background(), img, "", "", writer, nil, SigningOptions{}, false, false) + newImage, err := ir.New(context.Background(), img, "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) assert.NotEqual(t, newImage.ID(), "") err = newImage.Remove(false) @@ -163,7 +163,7 @@ func TestImage_MatchRepoTag(t *testing.T) { } ir, err := NewImageRuntimeFromOptions(so) assert.NoError(t, err) - newImage, err := ir.New(context.Background(), "busybox", "", "", os.Stdout, nil, SigningOptions{}, false, false) + newImage, err := ir.New(context.Background(), "busybox", "", "", os.Stdout, nil, SigningOptions{}, false) assert.NoError(t, err) err = newImage.TagImage("foo:latest") assert.NoError(t, err) diff --git a/libpod/image/pull.go b/libpod/image/pull.go index bfa04d069..09935fe7c 100644 --- a/libpod/image/pull.go +++ b/libpod/image/pull.go @@ -10,7 +10,6 @@ import ( "github.com/containers/image/directory" "github.com/containers/image/docker" dockerarchive "github.com/containers/image/docker/archive" - "github.com/containers/image/docker/reference" "github.com/containers/image/docker/tarfile" ociarchive "github.com/containers/image/oci/archive" "github.com/containers/image/pkg/sysregistries" @@ -19,7 +18,6 @@ import ( "github.com/containers/image/transports/alltransports" "github.com/containers/image/types" "github.com/containers/libpod/pkg/registries" - "github.com/containers/libpod/pkg/util" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -193,7 +191,7 @@ func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types. // pullImageFromHeuristicSource pulls an image based on inputName, which is heuristically parsed and may involve configured registries. // Use pullImageFromReference if the source is known precisely. -func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { var goal *pullGoal sc := GetSystemContext(signaturePolicyPath, authfile, false) srcRef, err := alltransports.ParseImageName(inputName) @@ -209,48 +207,33 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName) } } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, forceSecure) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions) } // pullImageFromReference pulls an image from a types.imageReference. -func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { sc := GetSystemContext(signaturePolicyPath, authfile, false) goal, err := ir.pullGoalFromImageReference(ctx, srcRef, transports.ImageName(srcRef), sc) if err != nil { return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef)) } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, forceSecure) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions) } // doPullImage is an internal helper interpreting pullGoal. Almost everyone should call one of the callers of doPullImage instead. -func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { policyContext, err := getPolicyContext(sc) if err != nil { return nil, err } defer policyContext.Destroy() - insecureRegistries, err := registries.GetInsecureRegistries() - if err != nil { - return nil, err - } + systemRegistriesConfPath := registries.SystemRegistriesConfPath() var images []string var pullErrors *multierror.Error for _, imageInfo := range goal.refPairs { copyOptions := getCopyOptions(sc, writer, dockerOptions, nil, signingOptions, "", nil) - if imageInfo.srcRef.Transport().Name() == DockerTransport { - imgRef := imageInfo.srcRef.DockerReference() - if imgRef == nil { // This should never happen; such references can’t be created. - return nil, fmt.Errorf("internal error: DockerTransport reference %s does not have a DockerReference", - transports.ImageName(imageInfo.srcRef)) - } - registry := reference.Domain(imgRef) - - if util.StringInSlice(registry, insecureRegistries) && !forceSecure { - copyOptions.SourceCtx.DockerInsecureSkipTLSVerify = true - logrus.Info(fmt.Sprintf("%s is an insecure registry; pulling with tls-verify=false", registry)) - } - } + copyOptions.SourceCtx.SystemRegistriesConfPath = systemRegistriesConfPath // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. // Print the following statement only when pulling from a docker or atomic registry if writer != nil && (imageInfo.srcRef.Transport().Name() == DockerTransport || imageInfo.srcRef.Transport().Name() == AtomicTransport) { io.WriteString(writer, fmt.Sprintf("Trying to pull %s...", imageInfo.image)) @@ -271,7 +254,7 @@ func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goa } // If no image was found, we should handle. Lets be nicer to the user and see if we can figure out why. if len(images) == 0 { - registryPath := sysregistries.RegistriesConfPath(&types.SystemContext{}) + registryPath := sysregistries.RegistriesConfPath(&types.SystemContext{SystemRegistriesConfPath: systemRegistriesConfPath}) if goal.usedSearchRegistries && len(goal.searchedRegistries) == 0 { return nil, errors.Errorf("image name provided is a short name and no search registries are defined in %s.", registryPath) } diff --git a/libpod/kube.go b/libpod/kube.go index 1a5f80878..05a6537c4 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -2,7 +2,10 @@ package libpod import ( "fmt" + "math/rand" + "strconv" "strings" + "time" "github.com/containers/libpod/pkg/lookup" "github.com/containers/libpod/pkg/util" @@ -15,23 +18,127 @@ import ( v12 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// InspectForKube takes a slice of libpod containers and generates +// GenerateForKube takes a slice of libpod containers and generates // one v1.Pod description that includes just a single container. -func (c *Container) InspectForKube() (*v1.Pod, error) { +func (c *Container) GenerateForKube() (*v1.Pod, error) { // Generate the v1.Pod yaml description return simplePodWithV1Container(c) } -// simplePodWithV1Container is a function used by inspect when kube yaml needs to be generated -// for a single container. we "insert" that container description in a pod. -func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) { - var containers []v1.Container - result, err := containerToV1Container(ctr) +// GenerateForKube takes a slice of libpod containers and generates +// one v1.Pod description +func (p *Pod) GenerateForKube() (*v1.Pod, []v1.ServicePort, error) { + // Generate the v1.Pod yaml description + var servicePorts []v1.ServicePort + + allContainers, err := p.allContainers() + if err != nil { + return nil, servicePorts, err + } + // If the pod has no containers, no sense to generate YAML + if len(allContainers) == 0 { + return nil, servicePorts, errors.Errorf("pod %s has no containers", p.ID()) + } + // If only an infra container is present, makes no sense to generate YAML + if len(allContainers) == 1 && p.HasInfraContainer() { + return nil, servicePorts, errors.Errorf("pod %s only has an infra container", p.ID()) + } + + if p.HasInfraContainer() { + infraContainer, err := p.getInfraContainer() + if err != nil { + return nil, servicePorts, err + } + + ports, err := ocicniPortMappingToContainerPort(infraContainer.config.PortMappings) + if err != nil { + return nil, servicePorts, err + } + servicePorts = containerPortsToServicePorts(ports) + } + pod, err := p.podWithContainers(allContainers) + return pod, servicePorts, err +} + +func (p *Pod) getInfraContainer() (*Container, error) { + infraID, err := p.InfraContainerID() if err != nil { return nil, err } - containers = append(containers, result) + return p.runtime.LookupContainer(infraID) +} + +// GenerateKubeServiceFromV1Pod creates a v1 service object from a v1 pod object +func GenerateKubeServiceFromV1Pod(pod *v1.Pod, servicePorts []v1.ServicePort) v1.Service { + service := v1.Service{} + selector := make(map[string]string) + selector["app"] = pod.Labels["app"] + ports := servicePorts + if len(ports) == 0 { + ports = containersToServicePorts(pod.Spec.Containers) + } + serviceSpec := v1.ServiceSpec{ + Ports: ports, + Selector: selector, + Type: v1.ServiceTypeNodePort, + } + service.Spec = serviceSpec + service.ObjectMeta = pod.ObjectMeta + tm := v12.TypeMeta{ + Kind: "Service", + APIVersion: pod.TypeMeta.APIVersion, + } + service.TypeMeta = tm + return service +} +// containerPortsToServicePorts takes a slice of containerports and generates a +// slice of service ports +func containerPortsToServicePorts(containerPorts []v1.ContainerPort) []v1.ServicePort { + var sps []v1.ServicePort + for _, cp := range containerPorts { + nodePort := 30000 + rand.Intn(32767-30000+1) + servicePort := v1.ServicePort{ + Protocol: cp.Protocol, + Port: cp.ContainerPort, + NodePort: int32(nodePort), + Name: strconv.Itoa(int(cp.ContainerPort)), + } + sps = append(sps, servicePort) + } + return sps +} + +// containersToServicePorts takes a slice of v1.Containers and generates an +// inclusive list of serviceports to expose +func containersToServicePorts(containers []v1.Container) []v1.ServicePort { + var sps []v1.ServicePort + // Without the call to rand.Seed, a program will produce the same sequence of pseudo-random numbers + // for each execution. Legal nodeport range is 30000-32767 + rand.Seed(time.Now().UnixNano()) + + for _, ctr := range containers { + sps = append(sps, containerPortsToServicePorts(ctr.Ports)...) + } + return sps +} + +func (p *Pod) podWithContainers(containers []*Container) (*v1.Pod, error) { + var podContainers []v1.Container + for _, ctr := range containers { + result, err := containerToV1Container(ctr) + if err != nil { + return nil, err + } + if !ctr.IsInfra() { + podContainers = append(podContainers, result) + } + } + + return addContainersToPodObject(podContainers, p.Name()), nil +} + +func addContainersToPodObject(containers []v1.Container, podName string) *v1.Pod { tm := v12.TypeMeta{ Kind: "Pod", APIVersion: "v1", @@ -39,10 +146,10 @@ func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) { // Add a label called "app" with the containers name as a value labels := make(map[string]string) - labels["app"] = removeUnderscores(ctr.Name()) + labels["app"] = removeUnderscores(podName) om := v12.ObjectMeta{ // The name of the pod is container_name-libpod - Name: fmt.Sprintf("%s-libpod", removeUnderscores(ctr.Name())), + Name: fmt.Sprintf("%s-libpod", removeUnderscores(podName)), Labels: labels, // CreationTimestamp seems to be required, so adding it; in doing so, the timestamp // will reflect time this is run (not container create time) because the conversion @@ -57,7 +164,20 @@ func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) { ObjectMeta: om, Spec: ps, } - return &p, nil + return &p +} + +// simplePodWithV1Container is a function used by inspect when kube yaml needs to be generated +// for a single container. we "insert" that container description in a pod. +func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) { + var containers []v1.Container + result, err := containerToV1Container(ctr) + if err != nil { + return nil, err + } + containers = append(containers, result) + return addContainersToPodObject(containers, ctr.Name()), nil + } // containerToV1Container converts information we know about a libpod container diff --git a/libpod/pod_api.go b/libpod/pod_api.go index 3d5512e8c..cbac2420f 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -62,7 +62,13 @@ func (p *Pod) Start(ctx context.Context) (map[string]error, error) { return nil, nil } -// Stop stops all containers within a pod that are not already stopped +// Stop stops all containers within a pod without a timeout. It assumes -1 for +// a timeout. +func (p *Pod) Stop(ctx context.Context, cleanup bool) (map[string]error, error) { + return p.StopWithTimeout(ctx, cleanup, -1) +} + +// StopWithTimeout stops all containers within a pod that are not already stopped // Each container will use its own stop timeout // Only running containers will be stopped. Paused, stopped, or created // containers will be ignored. @@ -77,7 +83,7 @@ func (p *Pod) Start(ctx context.Context) (map[string]error, error) { // containers. The container ID is mapped to the error encountered. The error is // set to ErrCtrExists // If both error and the map are nil, all containers were stopped without error -func (p *Pod) Stop(ctx context.Context, cleanup bool) (map[string]error, error) { +func (p *Pod) StopWithTimeout(ctx context.Context, cleanup bool, timeout int) (map[string]error, error) { p.lock.Lock() defer p.lock.Unlock() @@ -110,8 +116,11 @@ func (p *Pod) Stop(ctx context.Context, cleanup bool) (map[string]error, error) ctr.lock.Unlock() continue } - - if err := ctr.stop(ctr.config.StopTimeout); err != nil { + stopTimeout := ctr.config.StopTimeout + if timeout > -1 { + stopTimeout = uint(timeout) + } + if err := ctr.stop(stopTimeout); err != nil { ctr.lock.Unlock() ctrErrors[ctr.ID()] = err continue diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index be8711734..66844bb31 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -3,50 +3,15 @@ package libpod import ( "context" "fmt" - "io" "github.com/containers/buildah/imagebuildah" - "github.com/containers/libpod/libpod/common" "github.com/containers/libpod/libpod/image" "github.com/containers/storage" - "github.com/containers/storage/pkg/archive" - ociv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) // Runtime API -// CopyOptions contains the options given when pushing or pulling images -type CopyOptions struct { - // Compression specifies the type of compression which is applied to - // layer blobs. The default is to not use compression, but - // archive.Gzip is recommended. - Compression archive.Compression - // DockerRegistryOptions encapsulates settings that affect how we - // connect or authenticate to a remote registry to which we want to - // push the image. - common.DockerRegistryOptions - // SigningOptions encapsulates settings that control whether or not we - // strip or add signatures to the image when pushing (uploading) the - // image to a registry. - common.SigningOptions - - // SigningPolicyPath this points to a alternative signature policy file, used mainly for testing - SignaturePolicyPath string - // AuthFile is the path of the cached credentials file defined by the user - AuthFile string - // Writer is the reportWriter for the output - Writer io.Writer - // Reference is the name for the image created when a tar archive is imported - Reference string - // ImageConfig is the Image spec for the image created when a tar archive is imported - ImageConfig ociv1.Image - // ManifestMIMEType is the manifest type of the image when saving to a directory - ManifestMIMEType string - // ForceCompress compresses the image layers when saving to a directory using the dir transport if true - ForceCompress bool -} - // RemoveImage deletes an image from local storage // Images being used by running containers can only be removed if force=true func (r *Runtime) RemoveImage(ctx context.Context, img *image.Image, force bool) (string, error) { diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go index 8a5dbef56..5e1051150 100644 --- a/libpod/runtime_pod_infra_linux.go +++ b/libpod/runtime_pod_infra_linux.go @@ -67,7 +67,7 @@ func (r *Runtime) createInfraContainer(ctx context.Context, p *Pod) (*Container, return nil, ErrRuntimeStopped } - newImage, err := r.ImageRuntime().New(ctx, r.config.InfraImage, "", "", nil, nil, image.SigningOptions{}, false, false) + newImage, err := r.ImageRuntime().New(ctx, r.config.InfraImage, "", "", nil, nil, image.SigningOptions{}, false) if err != nil { return nil, err } diff --git a/libpod/util.go b/libpod/util.go index aa3494529..b7578135a 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -9,10 +9,8 @@ import ( "strings" "time" - "github.com/containerd/cgroups" "github.com/containers/image/signature" "github.com/containers/image/types" - "github.com/containers/libpod/pkg/util" "github.com/fsnotify/fsnotify" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -189,26 +187,3 @@ func validPodNSOption(p *Pod, ctrPod string) error { } return nil } - -// GetV1CGroups gets the V1 cgroup subsystems and then "filters" -// out any subsystems that are provided by the caller. Passing nil -// for excludes will return the subsystems unfiltered. -//func GetV1CGroups(excludes []string) ([]cgroups.Subsystem, error) { -func GetV1CGroups(excludes []string) cgroups.Hierarchy { - return func() ([]cgroups.Subsystem, error) { - var filtered []cgroups.Subsystem - - subSystem, err := cgroups.V1() - if err != nil { - return nil, err - } - for _, s := range subSystem { - // If the name of the subsystem is not in the list of excludes, then - // add it as a keeper. - if !util.StringInSlice(string(s.Name()), excludes) { - filtered = append(filtered, s) - } - } - return filtered, nil - } -} diff --git a/libpod/util_linux.go b/libpod/util_linux.go index 0cd486379..30e2538c3 100644 --- a/libpod/util_linux.go +++ b/libpod/util_linux.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containerd/cgroups" + "github.com/containers/libpod/pkg/util" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -67,3 +68,26 @@ func assembleSystemdCgroupName(baseSlice, newSlice string) (string, error) { return final, nil } + +// GetV1CGroups gets the V1 cgroup subsystems and then "filters" +// out any subsystems that are provided by the caller. Passing nil +// for excludes will return the subsystems unfiltered. +//func GetV1CGroups(excludes []string) ([]cgroups.Subsystem, error) { +func GetV1CGroups(excludes []string) cgroups.Hierarchy { + return func() ([]cgroups.Subsystem, error) { + var filtered []cgroups.Subsystem + + subSystem, err := cgroups.V1() + if err != nil { + return nil, err + } + for _, s := range subSystem { + // If the name of the subsystem is not in the list of excludes, then + // add it as a keeper. + if !util.StringInSlice(string(s.Name()), excludes) { + filtered = append(filtered, s) + } + } + return filtered, nil + } +} diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go index c26f15cb6..cbb8b730c 100644 --- a/pkg/registries/registries.go +++ b/pkg/registries/registries.go @@ -13,21 +13,28 @@ import ( // userRegistriesFile is the path to the per user registry configuration file. var userRegistriesFile = filepath.Join(os.Getenv("HOME"), ".config/containers/registries.conf") -// GetRegistries obtains the list of registries defined in the global registries file. -func GetRegistries() ([]string, error) { - registryConfigPath := "" +// SystemRegistriesConfPath returns an appropriate value for types.SystemContext.SystemRegistriesConfPath +// (possibly "", which is not an error), taking into account rootless mode and environment variable overrides. +// +// FIXME: This should be centralized in a global SystemContext initializer inherited throughout the code, +// not haphazardly called throughout the way it is being called now. +func SystemRegistriesConfPath() string { + if envOverride := os.Getenv("REGISTRIES_CONFIG_PATH"); len(envOverride) > 0 { + return envOverride + } if rootless.IsRootless() { if _, err := os.Stat(userRegistriesFile); err == nil { - registryConfigPath = userRegistriesFile + return userRegistriesFile } } - envOverride := os.Getenv("REGISTRIES_CONFIG_PATH") - if len(envOverride) > 0 { - registryConfigPath = envOverride - } - searchRegistries, err := sysregistries.GetRegistries(&types.SystemContext{SystemRegistriesConfPath: registryConfigPath}) + return "" +} + +// GetRegistries obtains the list of registries defined in the global registries file. +func GetRegistries() ([]string, error) { + searchRegistries, err := sysregistries.GetRegistries(&types.SystemContext{SystemRegistriesConfPath: SystemRegistriesConfPath()}) if err != nil { return nil, errors.Wrapf(err, "unable to parse the registries.conf file") } @@ -36,19 +43,7 @@ func GetRegistries() ([]string, error) { // GetInsecureRegistries obtains the list of insecure registries from the global registration file. func GetInsecureRegistries() ([]string, error) { - registryConfigPath := "" - - if rootless.IsRootless() { - if _, err := os.Stat(userRegistriesFile); err == nil { - registryConfigPath = userRegistriesFile - } - } - - envOverride := os.Getenv("REGISTRIES_CONFIG_PATH") - if len(envOverride) > 0 { - registryConfigPath = envOverride - } - registries, err := sysregistries.GetInsecureRegistries(&types.SystemContext{SystemRegistriesConfPath: registryConfigPath}) + registries, err := sysregistries.GetInsecureRegistries(&types.SystemContext{SystemRegistriesConfPath: SystemRegistriesConfPath()}) if err != nil { return nil, errors.Wrapf(err, "unable to parse the registries.conf file") } diff --git a/pkg/varlinkapi/containers_create.go b/pkg/varlinkapi/containers_create.go index f9a2db9c8..bb6273fd1 100644 --- a/pkg/varlinkapi/containers_create.go +++ b/pkg/varlinkapi/containers_create.go @@ -25,7 +25,7 @@ func (i *LibpodAPI) CreateContainer(call iopodman.VarlinkCall, config iopodman.C rtc := i.Runtime.GetConfig() ctx := getContext() - newImage, err := i.Runtime.ImageRuntime().New(ctx, config.Image, rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false, false) + newImage, err := i.Runtime.ImageRuntime().New(ctx, config.Image, rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false) if err != nil { return call.ReplyErrorOccurred(err.Error()) } diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index 6d3f19422..cb3b1c73b 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/containers/libpod/cmd/podman/shared" "io" "os" "path/filepath" @@ -16,6 +15,7 @@ import ( "github.com/containers/image/docker" "github.com/containers/image/manifest" "github.com/containers/image/types" + "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" @@ -322,13 +322,14 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, tlsVe destname = tag } - dockerRegistryOptions := image.DockerRegistryOptions{ - DockerInsecureSkipTLSVerify: !tlsVerify, + dockerRegistryOptions := image.DockerRegistryOptions{} + if !tlsVerify { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue } so := image.SigningOptions{} - if err := newImage.PushImageToHeuristicDestination(getContext(), destname, "", "", "", nil, false, so, &dockerRegistryOptions, false, nil); err != nil { + if err := newImage.PushImageToHeuristicDestination(getContext(), destname, "", "", "", nil, false, so, &dockerRegistryOptions, nil); err != nil { return call.ReplyErrorOccurred(err.Error()) } return call.ReplyPushImage(newImage.ID()) @@ -488,7 +489,7 @@ func (i *LibpodAPI) ExportImage(call iopodman.VarlinkCall, name, destination str return err } - if err := newImage.PushImageToHeuristicDestination(getContext(), destination, "", "", "", nil, compress, image.SigningOptions{}, &image.DockerRegistryOptions{}, false, additionalTags); err != nil { + if err := newImage.PushImageToHeuristicDestination(getContext(), destination, "", "", "", nil, compress, image.SigningOptions{}, &image.DockerRegistryOptions{}, additionalTags); err != nil { return call.ReplyErrorOccurred(err.Error()) } return call.ReplyExportImage(newImage.ID()) @@ -497,7 +498,7 @@ func (i *LibpodAPI) ExportImage(call iopodman.VarlinkCall, name, destination str // PullImage pulls an image from a registry to the image store. // TODO This implementation is incomplete func (i *LibpodAPI) PullImage(call iopodman.VarlinkCall, name string) error { - newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, "", "", nil, &image.DockerRegistryOptions{}, image.SigningOptions{}, true, false) + newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, "", "", nil, &image.DockerRegistryOptions{}, image.SigningOptions{}, true) if err != nil { return call.ReplyErrorOccurred(fmt.Sprintf("unable to pull %s: %s", name, err.Error())) } @@ -520,8 +521,10 @@ func (i *LibpodAPI) ImageExists(call iopodman.VarlinkCall, name string) error { func (i *LibpodAPI) ContainerRunlabel(call iopodman.VarlinkCall, input iopodman.Runlabel) error { ctx := getContext() dockerRegistryOptions := image.DockerRegistryOptions{ - DockerCertPath: input.CertDir, - DockerInsecureSkipTLSVerify: !input.TlsVerify, + DockerCertPath: input.CertDir, + } + if !input.TlsVerify { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue } stdErr := os.Stderr diff --git a/pkg/varlinkapi/pods.go b/pkg/varlinkapi/pods.go index 7930a956f..461371497 100644 --- a/pkg/varlinkapi/pods.go +++ b/pkg/varlinkapi/pods.go @@ -120,12 +120,12 @@ func (i *LibpodAPI) StartPod(call iopodman.VarlinkCall, name string) error { } // StopPod ... -func (i *LibpodAPI) StopPod(call iopodman.VarlinkCall, name string) error { +func (i *LibpodAPI) StopPod(call iopodman.VarlinkCall, name string, timeout int64) error { pod, err := i.Runtime.LookupPod(name) if err != nil { return call.ReplyPodNotFound(name) } - ctrErrs, err := pod.Stop(getContext(), true) + ctrErrs, err := pod.StopWithTimeout(getContext(), true, int(timeout)) callErr := handlePodCall(call, pod, ctrErrs, err) if callErr != nil { return err diff --git a/test/e2e/generate_kube_test.go b/test/e2e/generate_kube_test.go new file mode 100644 index 000000000..0ee078455 --- /dev/null +++ b/test/e2e/generate_kube_test.go @@ -0,0 +1,106 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + "github.com/ghodss/yaml" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman generate kube", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + + }) + + It("podman generate pod kube on bogus object", func() { + session := podmanTest.Podman([]string{"generate", "kube", "foobar"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) + + It("podman generate service kube on bogus object", func() { + session := podmanTest.Podman([]string{"generate", "kube", "-s", "foobar"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) + + It("podman generate kube on container", func() { + session := podmanTest.RunTopContainer("top") + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + kube := podmanTest.Podman([]string{"generate", "kube", "top"}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + _, err := yaml.Marshal(kube.OutputToString()) + Expect(err).To(BeNil()) + }) + + It("podman generate service kube on container", func() { + session := podmanTest.RunTopContainer("top") + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + kube := podmanTest.Podman([]string{"generate", "kube", "-s", "top"}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + _, err := yaml.Marshal(kube.OutputToString()) + Expect(err).To(BeNil()) + }) + + It("podman generate kube on pod", func() { + _, rc, _ := podmanTest.CreatePod("toppod") + Expect(rc).To(Equal(0)) + + session := podmanTest.RunTopContainerInPod("topcontainer", "toppod") + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + kube := podmanTest.Podman([]string{"generate", "kube", "toppod"}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + _, err := yaml.Marshal(kube.OutputToString()) + Expect(err).To(BeNil()) + }) + + It("podman generate service kube on pod", func() { + _, rc, _ := podmanTest.CreatePod("toppod") + Expect(rc).To(Equal(0)) + + session := podmanTest.RunTopContainerInPod("topcontainer", "toppod") + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + kube := podmanTest.Podman([]string{"generate", "kube", "-s", "toppod"}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + _, err := yaml.Marshal(kube.OutputToString()) + Expect(err).To(BeNil()) + }) +}) diff --git a/vendor.conf b/vendor.conf index 94eb6fccc..ac8a38355 100644 --- a/vendor.conf +++ b/vendor.conf @@ -11,7 +11,7 @@ github.com/containerd/cgroups 58556f5ad8448d99a6f7bea69ea4bdb7747cfeb0 github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 -github.com/containers/image bd10b1b53b2976f215b3f2f848fb8e7cad779aeb +github.com/containers/image 63a1cbdc5e6537056695cf0d627c0a33b334df53 github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 @@ -92,7 +92,7 @@ k8s.io/kube-openapi 275e2ce91dec4c05a4094a7b1daee5560b555ac9 https://github.com/ k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e https://github.com/kubernetes/utils github.com/mrunalp/fileutils master github.com/varlink/go master -github.com/containers/buildah 2ac987a52ff8412fb8f2908a191009751a6a1c62 +github.com/containers/buildah 9c65e5699cfa486531b3f123d9ce74873f0e18aa github.com/Nvveen/Gotty master github.com/fsouza/go-dockerclient master github.com/openshift/imagebuilder master diff --git a/vendor/github.com/containers/buildah/README.md b/vendor/github.com/containers/buildah/README.md index 2b539bba8..12eafdf88 100644 --- a/vendor/github.com/containers/buildah/README.md +++ b/vendor/github.com/containers/buildah/README.md @@ -105,6 +105,7 @@ $ sudo ./lighttpd.sh | [buildah-copy(1)](/docs/buildah-copy.md) | Copies the contents of a file, URL, or directory into a container's working directory. | | [buildah-from(1)](/docs/buildah-from.md) | Creates a new working container, either from scratch or using a specified image as a starting point. | | [buildah-images(1)](/docs/buildah-images.md) | List images in local storage. | +| [buildah-info(1)](/docs/buildah-info.md) | Display Buildah system information. | | [buildah-inspect(1)](/docs/buildah-inspect.md) | Inspects the configuration of a container or image. | | [buildah-mount(1)](/docs/buildah-mount.md) | Mount the working container's root filesystem. | | [buildah-pull(1)](/docs/buildah-pull.md) | Pull an image from the specified location. | diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 1a642ed3d..91ce2b09d 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -25,7 +25,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.5-dev" + Version = "1.6-dev" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/common.go b/vendor/github.com/containers/buildah/common.go index be59215df..dfdc33a22 100644 --- a/vendor/github.com/containers/buildah/common.go +++ b/vendor/github.com/containers/buildah/common.go @@ -38,7 +38,6 @@ func getCopyOptions(reportWriter io.Writer, sourceReference types.ImageReference if err != nil { logrus.Debugf("error determining if registry for %q is insecure: %v", transports.ImageName(sourceReference), err) } else if sourceInsecure { - sourceCtx.DockerInsecureSkipTLSVerify = true sourceCtx.OCIInsecureSkipTLSVerify = true } @@ -56,7 +55,6 @@ func getCopyOptions(reportWriter io.Writer, sourceReference types.ImageReference if err != nil { logrus.Debugf("error determining if registry for %q is insecure: %v", transports.ImageName(destinationReference), err) } else if destinationInsecure { - destinationCtx.DockerInsecureSkipTLSVerify = true destinationCtx.OCIInsecureSkipTLSVerify = true } diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 89224b674..3609694f6 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -543,3 +543,37 @@ func (b *Builder) SetStopSignal(stopSignal string) { b.OCIv1.Config.StopSignal = stopSignal b.Docker.Config.StopSignal = stopSignal } + +// Healthcheck returns information that recommends how a container engine +// should check if a running container is "healthy". +func (b *Builder) Healthcheck() *docker.HealthConfig { + if b.Docker.Config.Healthcheck == nil { + return nil + } + return &docker.HealthConfig{ + Test: copyStringSlice(b.Docker.Config.Healthcheck.Test), + Interval: b.Docker.Config.Healthcheck.Interval, + Timeout: b.Docker.Config.Healthcheck.Timeout, + StartPeriod: b.Docker.Config.Healthcheck.StartPeriod, + Retries: b.Docker.Config.Healthcheck.Retries, + } +} + +// SetHealthcheck sets recommended commands to run in order to verify that a +// running container based on this image is "healthy", along with information +// specifying how often that test should be run, and how many times the test +// should fail before the container should be considered unhealthy. +// Note: this setting is not present in the OCIv1 image format, so it is +// discarded when writing images using OCIv1 formats. +func (b *Builder) SetHealthcheck(config *docker.HealthConfig) { + b.Docker.Config.Healthcheck = nil + if config != nil { + b.Docker.Config.Healthcheck = &docker.HealthConfig{ + Test: copyStringSlice(config.Test), + Interval: config.Interval, + Timeout: config.Timeout, + StartPeriod: config.StartPeriod, + Retries: config.Retries, + } + } +} diff --git a/vendor/github.com/containers/buildah/docker/types.go b/vendor/github.com/containers/buildah/docker/types.go index 759fc1246..6847d36fd 100644 --- a/vendor/github.com/containers/buildah/docker/types.go +++ b/vendor/github.com/containers/buildah/docker/types.go @@ -60,8 +60,9 @@ type HealthConfig struct { Test []string `json:",omitempty"` // Zero means to inherit. Durations are expressed as integer nanoseconds. - Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. - Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. + Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + StartPeriod time.Duration `json:",omitempty"` // Time to wait after the container starts before running the first check. // Retries is the number of consecutive failures needed to consider a container as unhealthy. // Zero means inherit. diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index 701241683..e6ee6a071 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -15,6 +15,7 @@ import ( "time" "github.com/containers/buildah" + buildahdocker "github.com/containers/buildah/docker" "github.com/containers/buildah/util" cp "github.com/containers/image/copy" "github.com/containers/image/docker/reference" @@ -225,6 +226,18 @@ type Executor struct { copyFrom string // Used to keep track of the --from flag from COPY and ADD } +// builtinAllowedBuildArgs is list of built-in allowed build args +var builtinAllowedBuildArgs = map[string]bool{ + "HTTP_PROXY": true, + "http_proxy": true, + "HTTPS_PROXY": true, + "https_proxy": true, + "FTP_PROXY": true, + "ftp_proxy": true, + "NO_PROXY": true, + "no_proxy": true, +} + // withName creates a new child executor that will be used whenever a COPY statement uses --from=NAME. func (b *Executor) withName(name string, index int) *Executor { if b.named == nil { @@ -793,12 +806,28 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error commitName := b.output b.containerIDs = nil + var leftoverArgs []string + for arg := range b.builder.Args { + if !builtinAllowedBuildArgs[arg] { + leftoverArgs = append(leftoverArgs, arg) + } + } for i, node := range node.Children { step := ib.Step() if err := step.Resolve(node); err != nil { return errors.Wrapf(err, "error resolving step %+v", *node) } logrus.Debugf("Parsed Step: %+v", *step) + if step.Command == "arg" { + for index, arg := range leftoverArgs { + for _, Arg := range step.Args { + list := strings.SplitN(Arg, "=", 2) + if arg == list[0] { + leftoverArgs = append(leftoverArgs[:index], leftoverArgs[index+1:]...) + } + } + } + } if !b.quiet { b.log("%s", step.Original) } @@ -895,6 +924,9 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error } } } + if len(leftoverArgs) > 0 { + fmt.Fprintf(b.out, "[Warning] One or more build-args %v were not consumed\n", leftoverArgs) + } return nil } @@ -1139,6 +1171,17 @@ func (b *Executor) Commit(ctx context.Context, ib *imagebuilder.Builder, created b.builder.SetEntrypoint(config.Entrypoint) b.builder.SetShell(config.Shell) b.builder.SetStopSignal(config.StopSignal) + if config.Healthcheck != nil { + b.builder.SetHealthcheck(&buildahdocker.HealthConfig{ + Test: append([]string{}, config.Healthcheck.Test...), + Interval: config.Healthcheck.Interval, + Timeout: config.Healthcheck.Timeout, + StartPeriod: config.Healthcheck.StartPeriod, + Retries: config.Healthcheck.Retries, + }) + } else { + b.builder.SetHealthcheck(nil) + } b.builder.ClearLabels() for k, v := range config.Labels { b.builder.SetLabel(k, v) diff --git a/vendor/github.com/containers/buildah/info.go b/vendor/github.com/containers/buildah/info.go new file mode 100644 index 000000000..8cd5e4438 --- /dev/null +++ b/vendor/github.com/containers/buildah/info.go @@ -0,0 +1,207 @@ +package buildah + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "runtime" + "strconv" + "strings" + "time" + + "github.com/containers/libpod/pkg/rootless" + "github.com/containers/storage" + "github.com/containers/storage/pkg/system" + "github.com/sirupsen/logrus" +) + +// InfoData holds the info type, i.e store, host etc and the data for each type +type InfoData struct { + Type string + Data map[string]interface{} +} + +// Info returns the store and host information +func Info(store storage.Store) ([]InfoData, error) { + info := []InfoData{} + // get host information + hostInfo, err := hostInfo() + if err != nil { + logrus.Error(err, "error getting host info") + } + info = append(info, InfoData{Type: "host", Data: hostInfo}) + + // get store information + storeInfo, err := storeInfo(store) + if err != nil { + logrus.Error(err, "error getting store info") + } + info = append(info, InfoData{Type: "store", Data: storeInfo}) + return info, nil +} + +func hostInfo() (map[string]interface{}, error) { + info := map[string]interface{}{} + info["os"] = runtime.GOOS + info["arch"] = runtime.GOARCH + info["cpus"] = runtime.NumCPU() + info["rootless"] = rootless.IsRootless() + mi, err := system.ReadMemInfo() + if err != nil { + logrus.Error(err, "err reading memory info") + info["MemTotal"] = "" + info["MenFree"] = "" + info["SwapTotal"] = "" + info["SwapFree"] = "" + } else { + info["MemTotal"] = mi.MemTotal + info["MenFree"] = mi.MemFree + info["SwapTotal"] = mi.SwapTotal + info["SwapFree"] = mi.SwapFree + } + hostDistributionInfo := getHostDistributionInfo() + info["Distribution"] = map[string]interface{}{ + "distribution": hostDistributionInfo["Distribution"], + "version": hostDistributionInfo["Version"], + } + + kv, err := readKernelVersion() + if err != nil { + logrus.Error(err, "error reading kernel version") + } + info["kernel"] = kv + + up, err := readUptime() + if err != nil { + logrus.Error(err, "error reading up time") + } + // Convert uptime in seconds to a human-readable format + upSeconds := up + "s" + upDuration, err := time.ParseDuration(upSeconds) + if err != nil { + logrus.Error(err, "error parsing system uptime") + } + + hoursFound := false + var timeBuffer bytes.Buffer + var hoursBuffer bytes.Buffer + for _, elem := range upDuration.String() { + timeBuffer.WriteRune(elem) + if elem == 'h' || elem == 'm' { + timeBuffer.WriteRune(' ') + if elem == 'h' { + hoursFound = true + } + } + if !hoursFound { + hoursBuffer.WriteRune(elem) + } + } + + info["uptime"] = timeBuffer.String() + if hoursFound { + hours, err := strconv.ParseFloat(hoursBuffer.String(), 64) + if err == nil { + days := hours / 24 + info["uptime"] = fmt.Sprintf("%s (Approximately %.2f days)", info["uptime"], days) + } + } + + host, err := os.Hostname() + if err != nil { + logrus.Error(err, "error getting hostname") + } + info["hostname"] = host + + return info, nil + +} + +// top-level "store" info +func storeInfo(store storage.Store) (map[string]interface{}, error) { + // lets say storage driver in use, number of images, number of containers + info := map[string]interface{}{} + info["GraphRoot"] = store.GraphRoot() + info["RunRoot"] = store.RunRoot() + info["GraphDriverName"] = store.GraphDriverName() + info["GraphOptions"] = store.GraphOptions() + statusPairs, err := store.Status() + if err != nil { + return nil, err + } + status := map[string]string{} + for _, pair := range statusPairs { + status[pair[0]] = pair[1] + } + info["GraphStatus"] = status + images, err := store.Images() + if err != nil { + logrus.Error(err, "error getting number of images") + } + info["ImageStore"] = map[string]interface{}{ + "number": len(images), + } + + containers, err := store.Containers() + if err != nil { + logrus.Error(err, "error getting number of containers") + } + info["ContainerStore"] = map[string]interface{}{ + "number": len(containers), + } + + return info, nil +} + +func readKernelVersion() (string, error) { + buf, err := ioutil.ReadFile("/proc/version") + if err != nil { + return "", err + } + f := bytes.Fields(buf) + if len(f) < 2 { + return string(bytes.TrimSpace(buf)), nil + } + return string(f[2]), nil +} + +func readUptime() (string, error) { + buf, err := ioutil.ReadFile("/proc/uptime") + if err != nil { + return "", err + } + f := bytes.Fields(buf) + if len(f) < 1 { + return "", fmt.Errorf("invalid uptime") + } + return string(f[0]), nil +} + +// getHostDistributionInfo returns a map containing the host's distribution and version +func getHostDistributionInfo() map[string]string { + dist := make(map[string]string) + + // Populate values in case we cannot find the values + // or the file + dist["Distribution"] = "unknown" + dist["Version"] = "unknown" + + f, err := os.Open("/etc/os-release") + if err != nil { + return dist + } + defer f.Close() + + l := bufio.NewScanner(f) + for l.Scan() { + if strings.HasPrefix(l.Text(), "ID=") { + dist["Distribution"] = strings.TrimPrefix(l.Text(), "ID=") + } + if strings.HasPrefix(l.Text(), "VERSION_ID=") { + dist["Version"] = strings.Trim(strings.TrimPrefix(l.Text(), "VERSION_ID="), "\"") + } + } + return dist +} diff --git a/vendor/github.com/containers/buildah/pkg/parse/parse.go b/vendor/github.com/containers/buildah/pkg/parse/parse.go index b87eb95c7..41fdea8b1 100644 --- a/vendor/github.com/containers/buildah/pkg/parse/parse.go +++ b/vendor/github.com/containers/buildah/pkg/parse/parse.go @@ -282,7 +282,7 @@ func SystemContextFromOptions(c *cli.Context) (*types.SystemContext, error) { DockerCertPath: c.String("cert-dir"), } if c.IsSet("tls-verify") { - ctx.DockerInsecureSkipTLSVerify = !c.BoolT("tls-verify") + ctx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) ctx.OCIInsecureSkipTLSVerify = !c.BoolT("tls-verify") ctx.DockerDaemonInsecureSkipTLSVerify = !c.BoolT("tls-verify") } diff --git a/vendor/github.com/containers/buildah/util.go b/vendor/github.com/containers/buildah/util.go index 09aa7e1eb..66a4e535a 100644 --- a/vendor/github.com/containers/buildah/util.go +++ b/vendor/github.com/containers/buildah/util.go @@ -175,11 +175,11 @@ func (b *Builder) tarPath() func(path string) (io.ReadCloser, error) { // isRegistryInsecure checks if the named registry is marked as not secure func isRegistryInsecure(registry string, sc *types.SystemContext) (bool, error) { - registries, err := sysregistriesv2.GetRegistries(sc) + reginfo, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistries.RegistriesConfPath(sc)) } - if reginfo := sysregistriesv2.FindRegistry(registry, registries); reginfo != nil { + if reginfo != nil { if reginfo.Insecure { logrus.Debugf("registry %q is marked insecure in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) } else { @@ -193,11 +193,11 @@ func isRegistryInsecure(registry string, sc *types.SystemContext) (bool, error) // isRegistryBlocked checks if the named registry is marked as blocked func isRegistryBlocked(registry string, sc *types.SystemContext) (bool, error) { - registries, err := sysregistriesv2.GetRegistries(sc) + reginfo, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistries.RegistriesConfPath(sc)) } - if reginfo := sysregistriesv2.FindRegistry(registry, registries); reginfo != nil { + if reginfo != nil { if reginfo.Blocked { logrus.Debugf("registry %q is marked as blocked in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) } else { diff --git a/vendor/github.com/containers/buildah/util/util.go b/vendor/github.com/containers/buildah/util/util.go index b2451b78b..427c8db28 100644 --- a/vendor/github.com/containers/buildah/util/util.go +++ b/vendor/github.com/containers/buildah/util/util.go @@ -122,12 +122,11 @@ func ResolveName(name string, firstRegistry string, sc *types.SystemContext, sto // Figure out the list of registries. var registries []string - allRegistries, err := sysregistriesv2.GetRegistries(sc) + searchRegistries, err := sysregistriesv2.FindUnqualifiedSearchRegistries(sc) if err != nil { logrus.Debugf("unable to read configured registries to complete %q: %v", name, err) - registries = []string{} } - for _, registry := range sysregistriesv2.FindUnqualifiedSearchRegistries(allRegistries) { + for _, registry := range searchRegistries { if !registry.Blocked { registries = append(registries, registry.URL) } diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index 185cde449..acba0011e 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,7 +3,7 @@ github.com/blang/semver master github.com/BurntSushi/toml master github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 -github.com/containers/image de7be82ee3c7fb676bf6cfdc9090be7cc28f404c +github.com/containers/image 63a1cbdc5e6537056695cf0d627c0a33b334df53 github.com/containers/libpod fe4f09493f41f675d24c969d1b60d1a6a45ddb9e github.com/containers/storage 3161726d1db0d0d4e86a9667dd476f09b997f497 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 6d2c5b670..ea1a8ec06 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -17,6 +17,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/pkg/sysregistriesv2" "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" "github.com/docker/distribution/registry/client" @@ -78,11 +79,13 @@ type bearerToken struct { // dockerClient is configuration for dealing with a single Docker registry. type dockerClient struct { // The following members are set by newDockerClient and do not change afterwards. - sys *types.SystemContext - registry string + sys *types.SystemContext + registry string + client *http.Client + insecureSkipTLSVerify bool + // The following members are not set by newDockerClient and must be set by callers if needed. username string password string - client *http.Client signatureBase signatureStorageBase scope authScope // The following members are detected registry properties: @@ -194,13 +197,26 @@ func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write if err != nil { return nil, err } - remoteName := reference.Path(ref.ref) - return newDockerClientWithDetails(sys, registry, username, password, actions, sigBase, remoteName) + client, err := newDockerClient(sys, registry, ref.ref.Name()) + if err != nil { + return nil, err + } + client.username = username + client.password = password + client.signatureBase = sigBase + client.scope.actions = actions + client.scope.remoteName = reference.Path(ref.ref) + return client, nil } -// newDockerClientWithDetails returns a new dockerClient instance for the given parameters -func newDockerClientWithDetails(sys *types.SystemContext, registry, username, password, actions string, sigBase signatureStorageBase, remoteName string) (*dockerClient, error) { +// newDockerClient returns a new dockerClient instance for the given registry +// and reference. The reference is used to query the registry configuration +// and can either be a registry (e.g, "registry.com[:5000]"), a repository +// (e.g., "registry.com[:5000][/some/namespace]/repo"). +// Please note that newDockerClient does not set all members of dockerClient +// (e.g., username and password); those must be set by callers if necessary. +func newDockerClient(sys *types.SystemContext, registry, reference string) (*dockerClient, error) { hostName := registry if registry == dockerHostname { registry = dockerRegistry @@ -221,33 +237,43 @@ func newDockerClientWithDetails(sys *types.SystemContext, registry, username, pa return nil, err } - if sys != nil && sys.DockerInsecureSkipTLSVerify { - tr.TLSClientConfig.InsecureSkipVerify = true + // Check if TLS verification shall be skipped (default=false) which can + // either be specified in the sysregistriesv2 configuration or via the + // SystemContext, whereas the SystemContext is prioritized. + skipVerify := false + if sys != nil && sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined { + // Only use the SystemContext if the actual value is defined. + skipVerify = sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue + } else { + reg, err := sysregistriesv2.FindRegistry(sys, reference) + if err != nil { + return nil, errors.Wrapf(err, "error loading registries") + } + if reg != nil { + skipVerify = reg.Insecure + } } + tr.TLSClientConfig.InsecureSkipVerify = skipVerify return &dockerClient{ - sys: sys, - registry: registry, - username: username, - password: password, - client: &http.Client{Transport: tr}, - signatureBase: sigBase, - scope: authScope{ - actions: actions, - remoteName: remoteName, - }, + sys: sys, + registry: registry, + client: &http.Client{Transport: tr}, + insecureSkipTLSVerify: skipVerify, }, nil } // CheckAuth validates the credentials by attempting to log into the registry // returns an error if an error occcured while making the http request or the status code received was 401 func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password, registry string) error { - newLoginClient, err := newDockerClientWithDetails(sys, registry, username, password, "", nil, "") + client, err := newDockerClient(sys, registry, registry) if err != nil { return errors.Wrapf(err, "error creating new docker client") } + client.username = username + client.password = password - resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil, v2Auth) + resp, err := client.makeRequest(ctx, "GET", "/v2/", nil, nil, v2Auth) if err != nil { return err } @@ -299,16 +325,21 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima return nil, errors.Wrapf(err, "error getting username and password") } - // The /v2/_catalog endpoint has been disabled for docker.io therefore the call made to that endpoint will fail - // So using the v1 hostname for docker.io for simplicity of implementation and the fact that it returns search results + // The /v2/_catalog endpoint has been disabled for docker.io therefore + // the call made to that endpoint will fail. So using the v1 hostname + // for docker.io for simplicity of implementation and the fact that it + // returns search results. + hostname := registry if registry == dockerHostname { - registry = dockerV1Hostname + hostname = dockerV1Hostname } - client, err := newDockerClientWithDetails(sys, registry, username, password, "", nil, "") + client, err := newDockerClient(sys, hostname, registry) if err != nil { return nil, errors.Wrapf(err, "error creating new docker client") } + client.username = username + client.password = password // Only try the v1 search endpoint if the search query is not empty. If it is // empty skip to the v2 endpoint. @@ -530,7 +561,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { return nil } err := ping("https") - if err != nil && c.sys != nil && c.sys.DockerInsecureSkipTLSVerify { + if err != nil && c.insecureSkipTLSVerify { err = ping("http") } if err != nil { @@ -554,7 +585,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { return true } isV1 := pingV1("https") - if !isV1 && c.sys != nil && c.sys.DockerInsecureSkipTLSVerify { + if !isV1 && c.insecureSkipTLSVerify { isV1 = pingV1("http") } if isV1 { diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index 067f512ad..afc7312d1 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -3,7 +3,7 @@ package sysregistriesv2 import ( "fmt" "io/ioutil" - "net/url" + "os" "path/filepath" "strings" "sync" @@ -82,8 +82,8 @@ func (e *InvalidRegistries) Error() string { } // parseURL parses the input string, performs some sanity checks and returns -// the sanitized input string. An error is returned in case parsing fails or -// or if URI scheme or user is set. +// the sanitized input string. An error is returned if the input string is +// empty or if contains an "http{s,}://" prefix. func parseURL(input string) (string, error) { trimmed := strings.TrimRight(input, "/") @@ -91,49 +91,11 @@ func parseURL(input string) (string, error) { return "", &InvalidRegistries{s: "invalid URL: cannot be empty"} } - // Ultimately, we expect input of the form example.com[/namespace/…], a prefix - // of a fully-expended reference (containers/image/docker/Reference.String()). - // c/image/docker/Reference does not currently provide such a parser. - // So, we use url.Parse("http://"+trimmed) below to ~verify the format, possibly - // letting some invalid input in, trading that off for a simpler parser. - // - // url.Parse("http://"+trimmed) is, sadly, too permissive, notably for - // trimmed == "http://example.com/…", url.Parse("http://http://example.com/…") - // is accepted and parsed as - // {Scheme: "http", Host: "http:", Path: "//example.com/…"}. - // - // So, first we do an explicit check for an unwanted scheme prefix: - - // This will parse trimmed=="http://example.com/…" with Scheme: "http". Perhaps surprisingly, - // it also succeeds for the input we want to accept, in different ways: - // "example.com" -> {Scheme:"", Opaque:"", Path:"example.com"} - // "example.com/repo" -> {Scheme:"", Opaque:"", Path:"example.com/repo"} - // "example.com:5000" -> {Scheme:"example.com", Opaque:"5000"} - // "example.com:5000/repo" -> {Scheme:"example.com", Opaque:"5000/repo"} - uri, err := url.Parse(trimmed) - if err != nil { - return "", &InvalidRegistries{s: fmt.Sprintf("invalid URL '%s': %v", input, err)} - } - - // Check if a URI Scheme is set. - // Note that URLs that do not start with a slash after the scheme are - // interpreted as `scheme:opaque[?query][#fragment]`; see above for examples. - if uri.Scheme != "" && uri.Opaque == "" { + if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") { msg := fmt.Sprintf("invalid URL '%s': URI schemes are not supported", input) return "", &InvalidRegistries{s: msg} } - uri, err = url.Parse("http://" + trimmed) - if err != nil { - msg := fmt.Sprintf("invalid URL '%s': sanitized URL did not parse: %v", input, err) - return "", &InvalidRegistries{s: msg} - } - - if uri.User != nil { - msg := fmt.Sprintf("invalid URL '%s': user/password are not supported", trimmed) - return "", &InvalidRegistries{s: msg} - } - return trimmed, nil } @@ -279,7 +241,18 @@ var configMutex = sync.Mutex{} // are synchronized via configMutex. var configCache = make(map[string][]Registry) +// InvalidateCache invalidates the registry cache. This function is meant to be +// used for long-running processes that need to reload potential changes made to +// the cached registry config files. +func InvalidateCache() { + configMutex.Lock() + defer configMutex.Unlock() + configCache = make(map[string][]Registry) +} + // GetRegistries loads and returns the registries specified in the config. +// Note the parsed content of registry config files is cached. For reloading, +// use `InvalidateCache` and re-call `GetRegistries`. func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { configPath := getConfigPath(ctx) @@ -293,6 +266,13 @@ func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { // load the config config, err := loadRegistryConf(configPath) if err != nil { + // Return an empty []Registry if we use the default config, + // which implies that the config path of the SystemContext + // isn't set. Note: if ctx.SystemRegistriesConfPath points to + // the default config, we will still return an error. + if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") { + return []Registry{}, nil + } return nil, err } @@ -323,23 +303,33 @@ func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { // FindUnqualifiedSearchRegistries returns all registries that are configured // for unqualified image search (i.e., with Registry.Search == true). -func FindUnqualifiedSearchRegistries(registries []Registry) []Registry { +func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, error) { + registries, err := GetRegistries(ctx) + if err != nil { + return nil, err + } + unqualified := []Registry{} for _, reg := range registries { if reg.Search { unqualified = append(unqualified, reg) } } - return unqualified + return unqualified, nil } // FindRegistry returns the Registry with the longest prefix for ref. If no // Registry prefixes the image, nil is returned. -func FindRegistry(ref string, registries []Registry) *Registry { +func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { + registries, err := GetRegistries(ctx) + if err != nil { + return nil, err + } + reg := Registry{} prefixLen := 0 for _, r := range registries { - if strings.HasPrefix(ref, r.Prefix) { + if strings.HasPrefix(ref, r.Prefix+"/") || ref == r.Prefix { length := len(r.Prefix) if length > prefixLen { reg = r @@ -348,9 +338,9 @@ func FindRegistry(ref string, registries []Registry) *Registry { } } if prefixLen != 0 { - return ® + return ®, nil } - return nil + return nil, nil } // Reads the global registry file from the filesystem. Returns a byte array. diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 5d05b711a..a552e4597 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -324,6 +324,30 @@ type DockerAuthConfig struct { Password string } +// OptionalBool is a boolean with an additional undefined value, which is meant +// to be used in the context of user input to distinguish between a +// user-specified value and a default value. +type OptionalBool byte + +const ( + // OptionalBoolUndefined indicates that the OptionalBoolean hasn't been written. + OptionalBoolUndefined OptionalBool = iota + // OptionalBoolTrue represents the boolean true. + OptionalBoolTrue + // OptionalBoolFalse represents the boolean false. + OptionalBoolFalse +) + +// NewOptionalBool converts the input bool into either OptionalBoolTrue or +// OptionalBoolFalse. The function is meant to avoid boilerplate code of users. +func NewOptionalBool(b bool) OptionalBool { + o := OptionalBoolFalse + if b == true { + o = OptionalBoolTrue + } + return o +} + // SystemContext allows parameterizing access to implicitly-accessed resources, // like configuration files in /etc and users' login state in their home directory. // Various components can share the same field only if their semantics is exactly @@ -376,7 +400,7 @@ type SystemContext struct { // Ignored if DockerCertPath is non-empty. DockerPerHostCertDirPath string // Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. - DockerInsecureSkipTLSVerify bool + DockerInsecureSkipTLSVerify OptionalBool // if nil, the library tries to parse ~/.docker/config.json to retrieve credentials DockerAuthConfig *DockerAuthConfig // if not "", an User-Agent header is added to each request when contacting a registry. diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index 246c0096a..de6dcbecf 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -34,7 +34,7 @@ github.com/xeipuuv/gojsonschema master github.com/xeipuuv/gojsonreference master github.com/xeipuuv/gojsonpointer master github.com/tchap/go-patricia v2.2.6 -github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d +github.com/opencontainers/selinux 077c8b6d1c18456fb7c792bc0de52295a0d1900e github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 github.com/gogo/protobuf fcdc5011193ff531a548e9b0301828d5a5b97fd8 |