diff options
| -rw-r--r-- | .cirrus.yml | 46 | ||||
| -rw-r--r-- | contrib/cirrus/README.md | 47 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 17 | ||||
| -rwxr-xr-x | contrib/cirrus/optional_system_test.sh | 24 | ||||
| -rwxr-xr-x | contrib/cirrus/system_test.sh | 33 | ||||
| -rw-r--r-- | contrib/python/podman/podman/libs/_containers_attach.py | 8 | ||||
| -rw-r--r-- | contrib/python/pypodman/pypodman/lib/actions/__init__.py | 2 | ||||
| -rw-r--r-- | contrib/python/pypodman/pypodman/lib/actions/start_action.py | 76 | ||||
| -rw-r--r-- | contrib/python/pypodman/pypodman/lib/parser_actions.py | 29 | ||||
| -rw-r--r-- | libpod/runtime.go | 22 | ||||
| -rw-r--r-- | pkg/registries/registries.go | 6 | ||||
| -rw-r--r-- | pkg/secrets/secrets.go | 9 | ||||
| -rw-r--r-- | pkg/util/utils.go | 13 |
14 files changed, 241 insertions, 93 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f78205a49..c5b73fdc9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -63,24 +63,52 @@ full_vm_testing_task: integration_test_script: $SCRIPT_BASE/integration_test.sh - optional_system_test_script: $SCRIPT_BASE/optional_system_test.sh + success_script: $SCRIPT_BASE/success.sh + + +# Because system tests are stored within the repository, it is sometimes +# necessary to execute them within a PR to validate changes. + +optional_system_testing_task: + # Only run system tests in PRs (not on merge) if magic string is present + # in the PR description. Post-merge system testing is assumed to happen + # later from OS distribution's build systems. + only_if: >- + $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + + gce_instance: + matrix: + image_name: "ubuntu-1804-bionic-v20180911-libpod-63a86a18" + # TODO: Make these work (also build_images_task below) + #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" + #image_name: "centos-7-v20180911-libpod-fce09afe" + #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" + + timeout_in: 60m + + setup_environment_script: $SCRIPT_BASE/setup_environment.sh + system_test_script: $SCRIPT_BASE/system_test.sh success_script: $SCRIPT_BASE/success.sh -# This task build new images for future PR testing, but only after a PR merge. -# These images save needing to install/setup the same environment to test every -# PR. The 'active' image for testing is selected by the 'image_name' items in -# task above. Currently this requires manually updating them, but this could -# be automated (see comment at end). +# This task builds new cache-images for future PR testing. These images save +# time installing/setting up the environment while an engineer is waiting. +# The 'active' cache-images for full_vm_testing are selected by the +# 'image_name' keys. Updating those items requires manually modification, +# but this could be automated (see comment at end of build_vm_images_task). build_vm_images_task: - # Only produce new images after a PR merge - only_if: $CIRRUS_BRANCH == 'master' + # Only produce new cache-images after a PR merge, and if a magic string + # is present in the most recent commit-message. + only_if: >- + $CIRRUS_BRANCH == 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*' # Require tests to pass first. depends_on: - - test # i.e. 'test_task' + - full_vm_testing # i.e. 'full_vm_testing_task' env: # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json) diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 0d315c4f5..fa233a2cb 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -37,34 +37,41 @@ task (pass or fail) is set based on the exit status of the last script to execut Total execution time is capped at 2-hours (includes all the above) but this script normally completes in less than an hour. -### ``build_vm_images`` Task +### ``optional_system_testing`` Task -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another - round of the ``full_vm_testing`` task (above). +1. Optionally executes in parallel with ``full_vm_testing``. Requires + **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` + is found in the pull-request *description*. The *description* is the first + text-box under the main *summary* line in the github WebUI. -2. After confirming the tests all pass post-merge, spin up a special VM - capable of communicating with the GCE API. Once accessible, ``ssh`` into - the special VM and run the following scripts. +2. ``setup_environment.sh``: Same as for other tasks. -3. ``setup_environment.sh``: Configure root's ``.bash_profile`` - for all subsequent scripts (each run in a new shell). Any - distribution-specific environment variables are also defined - here. For example, setting tags/flags to use compiling. +3. ``system_test.sh``: Build both dependencies and libpod, install them, + then execute `make localsystem` from the repository root. + +### ``build_vm_images`` Task + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), Cirrus + checks the last commit message. If it contains the magic string + ``***CIRRUS: REBUILD IMAGES***``, then this task continues. + +2. Execute run another round of the ``full_vm_testing`` task (above). + After the tests pass (post-merge), spin up a special VM + (from the `image-builder-image`) capable of communicating with the + GCE API. Once accessible, ``ssh`` into the VM and run the following scripts. -4. ``build_vm_images.sh``: Examine the merged PR's description on github. - If it contains the magic string ``***CIRRUS: REBUILD IMAGES***``, then - continue. Otherwise display a message, take no further action, and - exit successfully. This prevents production of new VM images unless - they are called for, thereby saving the cost of needlessly storing them. +3. ``setup_environment.sh``: Same as for other tasks. -5. If the magic string was found, utilize [the packer tool](http://packer.io/docs/) +4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) to produce new VM images. Create a new VM from each base-image, connect - to them with ``ssh``, and perform these steps as defined by the - ``libpod_images.json`` file. + to them with ``ssh``, and perform the steps as defined by the + ``$PACKER_BASE/libpod_images.json`` file: - 1. Copy the current state of the repository into ``/tmp/libpod``. + 1. On a base-image VM, as root, copy the current state of the repository + into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``full_vm_testing`` task (above). + use by the ``full_vm_testing`` task (above). These scripts all + end with the suffix `_setup.sh` within the `$PACKER_BASE` directory. 3. If successful, shut down each VM and create a new GCE Image named after the base image and the commit sha of the merge. diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index ffbb2d5d5..c8ff55445 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -22,8 +22,6 @@ SCRIPT_BASE $SCRIPT_BASE PACKER_BASE $PACKER_BASE " -require_regex '\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*' 'Not re-building VM images' - show_env_vars # Everything here is running on the 'image-builder-image' GCE image diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 4a3efb8ff..6d43c6ea5 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -120,23 +120,6 @@ cdsudo() { sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD" } -# Skip a build if $1 does not match in the PR Title/Description with message $2 -require_regex() { - req_env_var " - CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE - 1 $1 - 2 $2 - " - regex="$1" - msg="$2" - if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$regex" - then - echo "***** The PR Title/Description did not match the regular expression: $MAGIC_RE" - echo "***** $msg" - exit 0 - fi -} - # Helper/wrapper script to only show stderr/stdout on non-zero exit install_ooe() { req_env_var "SCRIPT_BASE $SCRIPT_BASE" diff --git a/contrib/cirrus/optional_system_test.sh b/contrib/cirrus/optional_system_test.sh deleted file mode 100755 index 705dda5ad..000000000 --- a/contrib/cirrus/optional_system_test.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -e -source $(dirname $0)/lib.sh - -MAGIC_RE='\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*' -if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$MAGIC_RE" -then - echo "Skipping system-testing because PR title or description" - echo "does not match regular expression: $MAGIC_RE" - exit 0 -fi - -req_env_var " -GOSRC $GOSRC -OS_RELEASE_ID $OS_RELEASE_ID -OS_RELEASE_VER $OS_RELEASE_VER -" - -show_env_vars - -set -x -cd "$GOSRC" -make localsystem diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh new file mode 100755 index 000000000..7c727d336 --- /dev/null +++ b/contrib/cirrus/system_test.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e +source $(dirname $0)/lib.sh + +req_env_var " +GOSRC $GOSRC +OS_RELEASE_ID $OS_RELEASE_ID +OS_RELEASE_VER $OS_RELEASE_VER +" + +show_env_vars + +set -x +cd "$GOSRC" + +case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in + ubuntu-18) + make install.tools "BUILDTAGS=$BUILDTAGS" + make "BUILDTAGS=$BUILDTAGS" + make test-binaries "BUILDTAGS=$BUILDTAGS" + ;; + fedora-28) ;& + centos-7) ;& + rhel-7) + make install.tools + make + make test-binaries + ;; + *) bad_os_id_ver ;; +esac + +make localsystem diff --git a/contrib/python/podman/podman/libs/_containers_attach.py b/contrib/python/podman/podman/libs/_containers_attach.py index f2dad573b..94247d349 100644 --- a/contrib/python/podman/podman/libs/_containers_attach.py +++ b/contrib/python/podman/podman/libs/_containers_attach.py @@ -19,9 +19,13 @@ class Mixin: """ if stdin is None: stdin = sys.stdin.fileno() + elif hasattr(stdin, 'fileno'): + stdin = stdin.fileno() if stdout is None: stdout = sys.stdout.fileno() + elif hasattr(stdout, 'fileno'): + stdout = stdout.fileno() with self._client() as podman: attach = podman.GetAttachSockets(self._id) @@ -49,7 +53,7 @@ class Mixin: def resize_handler(self): """Send the new window size to conmon.""" - def wrapped(signum, frame): + def wrapped(signum, frame): # pylint: disable=unused-argument packed = fcntl.ioctl(self.pseudo_tty.stdout, termios.TIOCGWINSZ, struct.pack('HHHH', 0, 0, 0, 0)) rows, cols, _, _ = struct.unpack('HHHH', packed) @@ -67,7 +71,7 @@ class Mixin: def log_handler(self): """Send command to reopen log to conmon.""" - def wrapped(signum, frame): + def wrapped(signum, frame): # pylint: disable=unused-argument with open(self.pseudo_tty.control_socket, 'w') as skt: # send conmon reopen log message skt.write('2\n') diff --git a/contrib/python/pypodman/pypodman/lib/actions/__init__.py b/contrib/python/pypodman/pypodman/lib/actions/__init__.py index bc863ce6d..c0d77ddb1 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/__init__.py +++ b/contrib/python/pypodman/pypodman/lib/actions/__init__.py @@ -22,6 +22,7 @@ from pypodman.lib.actions.rm_action import Rm from pypodman.lib.actions.rmi_action import Rmi from pypodman.lib.actions.run_action import Run from pypodman.lib.actions.search_action import Search +from pypodman.lib.actions.start_action import Start from pypodman.lib.actions.version_action import Version __all__ = [ @@ -48,5 +49,6 @@ __all__ = [ 'Rmi', 'Run', 'Search', + 'Start', 'Version', ] diff --git a/contrib/python/pypodman/pypodman/lib/actions/start_action.py b/contrib/python/pypodman/pypodman/lib/actions/start_action.py new file mode 100644 index 000000000..f312fb3fa --- /dev/null +++ b/contrib/python/pypodman/pypodman/lib/actions/start_action.py @@ -0,0 +1,76 @@ +"""Remote client command for starting containers.""" +import sys + +import podman +from pypodman.lib import AbstractActionBase, BooleanAction + + +class Start(AbstractActionBase): + """Class for starting container.""" + + @classmethod + def subparser(cls, parent): + """Add Start command to parent parser.""" + parser = parent.add_parser('start', help='start container') + parser.add_argument( + '--attach', + '-a', + action=BooleanAction, + default=False, + help="Attach container's STDOUT and STDERR (default: %(default)s)") + parser.add_argument( + '--detach-keys', + metavar='KEY(s)', + default=4, + help='Override the key sequence for detaching a container.' + ' (format: a single character [a-Z] or ctrl-<value> where' + ' <value> is one of: a-z, @, ^, [, , or _) (default: ^D)') + parser.add_argument( + '--interactive', + '-i', + action=BooleanAction, + default=False, + help="Attach container's STDIN (default: %(default)s)") + # TODO: Implement sig-proxy + parser.add_argument( + '--sig-proxy', + action=BooleanAction, + default=False, + help="Proxy received signals to the process (default: %(default)s)" + ) + parser.add_argument( + 'containers', + nargs='+', + help='containers to start', + ) + parser.set_defaults(class_=cls, method='start') + + def start(self): + """Start provided containers.""" + stdin = sys.stdin if self.opts['interactive'] else None + stdout = sys.stdout if self.opts['attach'] else None + + try: + for ident in self._args.containers: + try: + ctnr = self.client.containers.get(ident) + ctnr.attach( + eot=self.opts['detach_keys'], + stdin=stdin, + stdout=stdout) + ctnr.start() + except podman.ContainerNotFound as e: + sys.stdout.flush() + print( + 'Container "{}" not found'.format(e.name), + file=sys.stderr, + flush=True) + else: + print(ident) + except podman.ErrorOccurred as e: + sys.stdout.flush() + print( + '{}'.format(e.reason).capitalize(), + file=sys.stderr, + flush=True) + return 1 diff --git a/contrib/python/pypodman/pypodman/lib/parser_actions.py b/contrib/python/pypodman/pypodman/lib/parser_actions.py index c10b85495..77ee14761 100644 --- a/contrib/python/pypodman/pypodman/lib/parser_actions.py +++ b/contrib/python/pypodman/pypodman/lib/parser_actions.py @@ -37,7 +37,7 @@ class BooleanAction(argparse.Action): const=None, default=None, type=None, - choices=('True', 'False'), + choices=None, required=False, help=None, metavar='{True,False}'): @@ -59,7 +59,7 @@ class BooleanAction(argparse.Action): try: val = BooleanValidate()(values) except ValueError: - parser.error('{} must be True or False.'.format(self.dest)) + parser.error('"{}" must be True or False.'.format(option_string)) else: setattr(namespace, self.dest, val) @@ -96,7 +96,6 @@ class ChangeAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): """Convert and Validate input.""" - print(self.dest) items = getattr(namespace, self.dest, None) or [] items = copy.copy(items) @@ -105,9 +104,9 @@ class ChangeAction(argparse.Action): opt, val = values.split('=', 1) if opt not in choices: - parser.error('{} is not a supported "--change" option,' + parser.error('Option "{}" is not supported by argument "{}",' ' valid options are: {}'.format( - opt, ', '.join(choices))) + opt, option_string, ', '.join(choices))) items.append(values) setattr(namespace, self.dest, items) @@ -127,8 +126,8 @@ class UnitAction(argparse.Action): help=None, metavar='UNIT'): """Create UnitAction object.""" - help = (help or metavar or dest - ) + ' (format: <number>[<unit>], where unit = b, k, m or g)' + help = (help or metavar or dest)\ + + ' (format: <number>[<unit>], where unit = b, k, m or g)' super().__init__( option_strings=option_strings, dest=dest, @@ -148,15 +147,15 @@ class UnitAction(argparse.Action): except ValueError: if not values[:-1].isdigit(): msg = ('{} must be a positive integer,' - ' with optional suffix').format(self.dest) + ' with optional suffix').format(option_string) parser.error(msg) if not values[-1] in ('b', 'k', 'm', 'g'): msg = '{} only supports suffices of: b, k, m, g'.format( - self.dest) + option_string) parser.error(msg) else: if val <= 0: - msg = '{} must be a positive integer'.format(self.dest) + msg = '{} must be a positive integer'.format(option_string) parser.error(msg) setattr(namespace, self.dest, values) @@ -174,19 +173,16 @@ class PositiveIntAction(argparse.Action): type=int, choices=None, required=False, - help=None, + help='Must be a positive integer.', metavar=None): """Create PositiveIntAction object.""" - self.message = '{} must be a positive integer'.format(dest) - help = help or self.message - super().__init__( option_strings=option_strings, dest=dest, nargs=nargs, const=const, default=default, - type=int, + type=type, choices=choices, required=required, help=help, @@ -198,7 +194,8 @@ class PositiveIntAction(argparse.Action): setattr(namespace, self.dest, values) return - parser.error(self.message) + msg = '{} must be a positive integer'.format(option_string) + parser.error(msg) class PathAction(argparse.Action): diff --git a/libpod/runtime.go b/libpod/runtime.go index 318cd0369..9feae03fc 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -264,6 +264,7 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { configPath := ConfigPath foundConfig := true + rootlessConfigPath := "" if rootless.IsRootless() { home := os.Getenv("HOME") if runtime.config.SignaturePolicyPath == "" { @@ -272,7 +273,10 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { runtime.config.SignaturePolicyPath = newPath } } - configPath = filepath.Join(home, ".config/containers/libpod.conf") + + rootlessConfigPath = filepath.Join(home, ".config/containers/libpod.conf") + + configPath = rootlessConfigPath if _, err := os.Stat(configPath); err != nil { foundConfig = false } @@ -317,6 +321,22 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { if err := makeRuntime(runtime); err != nil { return nil, err } + + if !foundConfig && rootlessConfigPath != "" { + os.MkdirAll(filepath.Dir(rootlessConfigPath), 0755) + file, err := os.OpenFile(rootlessConfigPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil && !os.IsExist(err) { + return nil, errors.Wrapf(err, "cannot open file %s", rootlessConfigPath) + } + if err == nil { + defer file.Close() + enc := toml.NewEncoder(file) + if err := enc.Encode(runtime.config); err != nil { + os.Remove(rootlessConfigPath) + } + } + } + return runtime, nil } diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go index 73aa93d68..c26f15cb6 100644 --- a/pkg/registries/registries.go +++ b/pkg/registries/registries.go @@ -38,8 +38,10 @@ func GetRegistries() ([]string, error) { func GetInsecureRegistries() ([]string, error) { registryConfigPath := "" - if _, err := os.Stat(userRegistriesFile); err == nil { - registryConfigPath = userRegistriesFile + if rootless.IsRootless() { + if _, err := os.Stat(userRegistriesFile); err == nil { + registryConfigPath = userRegistriesFile + } } envOverride := os.Getenv("REGISTRIES_CONFIG_PATH") diff --git a/pkg/secrets/secrets.go b/pkg/secrets/secrets.go index 7208f53b7..242953609 100644 --- a/pkg/secrets/secrets.go +++ b/pkg/secrets/secrets.go @@ -149,6 +149,15 @@ func SecretMountsWithUIDGID(mountLabel, containerWorkingDir, mountFile, mountPre mountFiles = append(mountFiles, []string{OverrideMountsFile, DefaultMountsFile}...) if rootless.IsRootless() { mountFiles = append([]string{UserOverrideMountsFile}, mountFiles...) + _, err := os.Stat(UserOverrideMountsFile) + if err != nil && os.IsNotExist(err) { + os.MkdirAll(filepath.Dir(UserOverrideMountsFile), 0755) + if f, err := os.Create(UserOverrideMountsFile); err != nil { + logrus.Warnf("could not create file %s: %v", UserOverrideMountsFile, err) + } else { + f.Close() + } + } } } else { mountFiles = append(mountFiles, mountFile) diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 3b43489b2..c5ba38b9f 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -9,6 +9,7 @@ import ( "strings" "syscall" + "github.com/BurntSushi/toml" "github.com/containers/image/types" "github.com/containers/libpod/pkg/rootless" "github.com/containers/storage" @@ -296,6 +297,18 @@ func GetDefaultStoreOptions() (storage.StoreOptions, error) { storageConf := filepath.Join(os.Getenv("HOME"), ".config/containers/storage.conf") if _, err := os.Stat(storageConf); err == nil { storage.ReloadConfigurationFile(storageConf, &storageOpts) + } else if os.IsNotExist(err) { + os.MkdirAll(filepath.Dir(storageConf), 0755) + file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf) + } + + defer file.Close() + enc := toml.NewEncoder(file) + if err := enc.Encode(storageOpts); err != nil { + os.Remove(storageConf) + } } } return storageOpts, nil |