diff options
| -rw-r--r-- | .cirrus.yml | 20 | ||||
| -rwxr-xr-x | API.md | 2 | ||||
| -rw-r--r-- | Dockerfile.fedora | 1 | ||||
| -rw-r--r-- | cmd/podman/shared/create.go | 2 | ||||
| -rw-r--r-- | cmd/podman/varlink/io.podman.varlink | 4 | ||||
| -rwxr-xr-x | contrib/imgprune/entrypoint.sh | 63 | ||||
| -rw-r--r-- | libpod/image/image.go | 6 | ||||
| -rw-r--r-- | pkg/varlinkapi/images.go | 15 | ||||
| -rw-r--r-- | test/system/065-cp.bats | 229 |
9 files changed, 315 insertions, 27 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f034a5b37..f49b1d312 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -270,6 +270,7 @@ meta_task: BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df] + # needed for output-masking purposes GCPNAME: ENCRYPTED[b05d469a0dba8cb479cb00cc7c1f6747c91d17622fba260a986b976aa6c817d4077eacffd4613d6d5f23afc4084fab1d] GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] @@ -294,9 +295,11 @@ image_prune_task: memory: 1 env: - <<: *meta_env_vars + # order is significant, Cirrus not always overriding alias values as intended GCPJSON: ENCRYPTED[4c11d8e09c904c30fc70eecb95c73dec0ddf19976f9b981a0f80f3f6599e8f990bcef93c253ac0277f200850d98528e7] GCPNAME: ENCRYPTED[7f54557ba6e5a437f11283a53e71baec9ca546f48a9835538cc54d297f79968eb1337d4596a1025b14f9d1c5723fbd29] + GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + <<: *meta_env_vars timeout_in: 10m @@ -525,6 +528,19 @@ special_testing_endpoint_task: always: <<: *standardlogs + +test_building_snap_task: + + depends_on: + - "gating" + + container: + image: yakshaveinc/snapcraft:core18 + snapcraft_script: + - 'apt-get -y update' + - 'cd contrib/snapcraft && snapcraft' + + # Test building of new cache-images for future PR testing, in this PR. test_build_cache_images_task: @@ -633,6 +649,7 @@ success_task: - "special_testing_cross" - "special_testing_endpoint" - "test_build_cache_images" + - "test_building_snap" - "verify_test_built_images" env: @@ -674,6 +691,7 @@ release_task: - "special_testing_cross" - "special_testing_endpoint" - "test_build_cache_images" + - "test_building_snap" - "verify_test_built_images" - "success" @@ -1361,6 +1361,8 @@ pids [int](https://godoc.org/builtin#int) ### <a name="Create"></a>type Create Create is an input structure for creating containers. +args[0] is the image name or id +args[1-] are the new commands if changed args [[]string](#[]string) diff --git a/Dockerfile.fedora b/Dockerfile.fedora index 429597f4f..9b1568b0b 100644 --- a/Dockerfile.fedora +++ b/Dockerfile.fedora @@ -28,6 +28,7 @@ RUN dnf -y install btrfs-progs-devel \ slirp4netns \ container-selinux \ containernetworking-plugins \ + iproute \ iptables && dnf clean all # Install ginkgo diff --git a/cmd/podman/shared/create.go b/cmd/podman/shared/create.go index 094330e24..acbd53dba 100644 --- a/cmd/podman/shared/create.go +++ b/cmd/podman/shared/create.go @@ -81,7 +81,7 @@ func CreateContainer(ctx context.Context, c *GenericCLIResults, runtime *libpod. if len(c.InputArgs) != 0 { name = c.InputArgs[0] } else { - return nil, nil, errors.Errorf("error, no input arguments were provided") + return nil, nil, errors.Errorf("error, image name not provided") } pullType, err := util.ValidatePullType(c.String("pull")) diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index 2873d9761..752e28256 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -274,6 +274,8 @@ type Sockets( ) # Create is an input structure for creating containers. +# args[0] is the image name or id +# args[1-] are the new commands if changed type Create ( args: []string, addHost: ?[]string, @@ -1281,4 +1283,4 @@ error WantsMoreRequired (reason: string) error ErrCtrStopped (id: string) # This function requires CGroupsV2 to run in rootless mode. -error ErrRequiresCgroupsV2ForRootless(reason: string)
\ No newline at end of file +error ErrRequiresCgroupsV2ForRootless(reason: string) diff --git a/contrib/imgprune/entrypoint.sh b/contrib/imgprune/entrypoint.sh index a4b77523b..829e9938e 100755 --- a/contrib/imgprune/entrypoint.sh +++ b/contrib/imgprune/entrypoint.sh @@ -6,27 +6,49 @@ source /usr/local/bin/lib_entrypoint.sh req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES +BASE_IMAGES="" +# When executing under Cirrus-CI, have access to current source +if [[ "$CI" == "true" ]] && [[ -r "$CIRRUS_WORKING_DIR/$SCRIPT_BASE" ]] +then + # Avoid importing anything that might conflict + eval "$(egrep -sh '^export .+BASE_IMAGE=' < $CIRRUS_WORKING_DIR/$SCRIPT_BASE/lib.sh)" + BASE_IMAGES="$UBUNTU_BASE_IMAGE $PRIOR_UBUNTU_BASE_IMAGE $FEDORA_BASE_IMAGE $PRIOR_FEDORA_BASE_IMAGE" +else + # metadata labeling may have broken for some reason in the future + echo "Warning: Running outside of Cirrus-CI, very minor-risk of base-image deletion." +fi + gcloud_init # For safety's sake + limit nr background processes -PRUNE_LIMIT=10 +PRUNE_LIMIT=5 THEFUTURE=$(date --date='+1 hour' +%s) -TOO_OLD='90 days ago' +TOO_OLD='30 days ago' THRESHOLD=$(date --date="$TOO_OLD" +%s) # Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats FORMAT='value[quote](name,selfLink,creationTimestamp,labels)' PROJRE="/v1/projects/$GCPPROJECT/global/" -BASE_IMAGE_RE='cloud-base' -RECENTLY=$(date --date='30 days ago' --iso-8601=date) -EXCLUDE="$IMGNAMES $IMAGE_BUILDER_CACHE_IMAGE_NAME" # whitespace separated values +RECENTLY=$(date --date='3 days ago' --iso-8601=date) # Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters -FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($EXCLUDE)" +FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($IMGNAMES $BASE_IMAGES)" TODELETE=$(mktemp -p '' todelete.XXXXXX) +IMGCOUNT=$(mktemp -p '' imgcount.XXXXXX) + +# Search-loop runs in a sub-process, must store count in file +echo "0" > "$IMGCOUNT" +count_image() { + local count + count=$(<"$IMGCOUNT") + let 'count+=1' + echo "$count" > "$IMGCOUNT" +} -echo "Searching images for pruning candidates older than $TOO_OLD ($THRESHOLD):" +echo "Using filter: $FILTER" +echo "Searching images for pruning candidates older than $TOO_OLD ($(date --date="$TOO_OLD" --iso-8601=date)):" $GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \ while read name selfLink creationTimestamp labels do + count_image created_ymd=$(date --date=$creationTimestamp --iso-8601=date) last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true) markmsgpfx="Marking $name (created $created_ymd) for deletion" @@ -52,16 +74,29 @@ $GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \ echo "$name" >> $TODELETE continue fi - - echo "NOT $markmsgpfx: last used on $last_used_ymd)" done -echo "Pruning up to $PRUNE_LIMIT images that were marked for deletion:" -for image_name in $(tail -$PRUNE_LIMIT $TODELETE | sort --random-sort) +COUNT=$(<"$IMGCOUNT") +echo "########################################################################" +echo "Deleting up to $PRUNE_LIMIT images marked ($(wc -l < $TODELETE)) of all searched ($COUNT):" + +# Require a minimum number of images to exist +NEED="$[$PRUNE_LIMIT*2]" +if [[ "$COUNT" -lt "$NEED" ]] +then + die 0 Safety-net Insufficient images \($COUNT\) to process deletions \($NEED\) + exit 0 +fi + +for image_name in $(sort --random-sort $TODELETE | tail -$PRUNE_LIMIT) do - # This can take quite some time (minutes), run in parallel disconnected from terminal - echo "TODO: Would have: $GCLOUD compute images delete $image_name &" - sleep "$[1+RANDOM/1000]s" & # Simlate background operation + if echo "$IMGNAMES $BASE_IMAGES" | grep -q "$image_name" + then + # double-verify in-use images were filtered out in search loop above + die 8 FATAL ATTEMPT TO DELETE IN-USE IMAGE \'$image_name\' - THIS SHOULD NEVER HAPPEN + fi + echo "Deleting $image_name in parallel..." + $GCLOUD compute images delete $image_name & done wait || true # Nothing to delete: No background jobs diff --git a/libpod/image/image.go b/libpod/image/image.go index 1ff271a4d..0be6eeeb9 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -12,7 +12,6 @@ import ( "syscall" "time" - types2 "github.com/containernetworking/cni/pkg/types" cp "github.com/containers/image/copy" "github.com/containers/image/directory" dockerarchive "github.com/containers/image/docker/archive" @@ -384,11 +383,6 @@ func (i *Image) Remove(ctx context.Context, force bool) error { return nil } -// Decompose an Image -func (i *Image) Decompose() error { - return types2.NotImplementedError -} - // TODO: Rework this method to not require an assembly of the fq name with transport /* // GetManifest tries to GET an images manifest, returns nil on success and err on failure diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index 0e1ef5b76..c184155a9 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -338,7 +338,7 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr dockerRegistryOptions := image.DockerRegistryOptions{} if format != "" { switch format { - case "oci": //nolint + case "oci": // nolint manifestType = v1.MediaTypeImageManifest case "v2s1": manifestType = manifest.DockerV2Schema1SignedMediaType @@ -360,7 +360,12 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr output := bytes.NewBuffer([]byte{}) c := make(chan error) go func() { - err := newImage.PushImageToHeuristicDestination(getContext(), destname, manifestType, "", "", "", output, compress, so, &dockerRegistryOptions, nil) + writer := bytes.NewBuffer([]byte{}) + err := newImage.PushImageToHeuristicDestination(getContext(), destname, manifestType, "", "", "", writer, compress, so, &dockerRegistryOptions, nil) + if err != nil { + c <- err + } + _, err = io.CopyBuffer(output, writer, nil) c <- err close(c) }() @@ -388,6 +393,7 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr } br := iopodman.MoreResponse{ Logs: log, + Id: newImage.ID(), } call.ReplyPushImage(br) log = []string{} @@ -403,6 +409,7 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr br := iopodman.MoreResponse{ Logs: log, + Id: newImage.ID(), } return call.ReplyPushImage(br) } @@ -530,7 +537,7 @@ func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, ch } sc := image.GetSystemContext(rtc.SignaturePolicyPath, "", false) switch manifestType { - case "oci", "": //nolint + case "oci", "": // nolint mimeType = buildah.OCIv1ImageManifest case "docker": mimeType = manifest.DockerV2Schema2MediaType @@ -821,7 +828,7 @@ func (i *LibpodAPI) ImageSave(call iopodman.VarlinkCall, options iopodman.ImageS // Image has been saved to `output` if outputToDir { // If the output is a directory, we need to tar up the directory to send it back - //Create a tempfile for the directory tarball + // Create a tempfile for the directory tarball outputFile, err := ioutil.TempFile("", "varlink_save_dir") if err != nil { return err diff --git a/test/system/065-cp.bats b/test/system/065-cp.bats new file mode 100644 index 000000000..204065bdb --- /dev/null +++ b/test/system/065-cp.bats @@ -0,0 +1,229 @@ +#!/usr/bin/env bats -*- bats -*- +# +# Tests for 'podman cp' +# +# ASSUMPTION FOR ALL THESE TESTS: /tmp in the container starts off empty +# + +load helpers + +# Create two random-name random-content files in /tmp in the container +# podman-cp them into the host using '/tmp/*', i.e. asking podman to +# perform wildcard expansion in the container. We should get both +# files copied into the host. +@test "podman cp * - wildcard copy multiple files from container to host" { + skip_if_remote "podman-remote does not yet handle cp" + + srcdir=$PODMAN_TMPDIR/cp-test-in + dstdir=$PODMAN_TMPDIR/cp-test-out + mkdir -p $srcdir $dstdir + + rand_filename1=$(random_string 20) + rand_content1=$(random_string 50) + rand_filename2=$(random_string 20) + rand_content2=$(random_string 50) + + run_podman run --name cpcontainer $IMAGE sh -c \ + "echo $rand_content1 >/tmp/$rand_filename1; + echo $rand_content2 >/tmp/$rand_filename2" + + run_podman cp 'cpcontainer:/tmp/*' $dstdir + + test -e $dstdir/$rand_filename1 || die "file 1 not copied from container" + test -e $dstdir/$rand_filename2 || die "file 2 not copied from container" + + is "$(<$dstdir/$rand_filename1)" "$rand_content1" "content of file 1" + is "$(<$dstdir/$rand_filename2)" "$rand_content2" "content of file 2" + + run_podman rm cpcontainer +} + + +# Create a file on the host; make a symlink in the container pointing +# into host-only space. Try to podman-cp that symlink. It should fail. +@test "podman cp - will not recognize symlink pointing into host space" { + skip_if_remote "podman-remote does not yet handle cp" + skip "BROKEN: PLEASE ENABLE ONCE #3829 GETS FIXED" + + srcdir=$PODMAN_TMPDIR/cp-test-in + dstdir=$PODMAN_TMPDIR/cp-test-out + mkdir -p $srcdir $dstdir + echo "this file is on the host" >$srcdir/hostfile + + run_podman run --name cpcontainer $IMAGE \ + sh -c "ln -s $srcdir/hostfile /tmp/badlink" + # This should fail because, from the container's perspective, the symlink + # points to a nonexistent file + run_podman 125 cp 'cpcontainer:/tmp/*' $dstdir/ + + # FIXME: this might not be the exactly correct error message + is "$output" ".*error evaluating symlinks.*lstat.*no such file or dir" \ + "Expected error from copying invalid symlink" + + # make sure there are no files in dstdir + is "$(/bin/ls -1 $dstdir)" "" "incorrectly copied symlink from host" + + run_podman rm cpcontainer +} + + +# Issue #3829 - like the above, but with a level of indirection in the +# wildcard expansion: create a file on the host; create a symlink in +# the container named 'file1' pointing to this file; then another symlink +# in the container pointing to 'file*' (file star). Try to podman-cp +# this invalid double symlink. It must fail. +@test "podman cp - will not expand globs in host space (#3829)" { + skip_if_remote "podman-remote does not yet handle cp" + skip "BROKEN: PLEASE ENABLE ONCE #3829 GETS FIXED" + + srcdir=$PODMAN_TMPDIR/cp-test-in + dstdir=$PODMAN_TMPDIR/cp-test-out + mkdir -p $srcdir $dstdir + echo "This file is on the host" > $srcdir/hostfile + + run_podman run --name cpcontainer $IMAGE \ + sh -c "ln -s $srcdir/hostfile file1;ln -s file\* copyme" + run_podman 125 cp cpcontainer:copyme $dstdir + + is "$output" ".*error evaluating symlinks.*lstat.*no such file or dir" \ + "Expected error from copying invalid symlink" + + # make sure there are no files in dstdir + is "$(/bin/ls -1 $dstdir)" "" "incorrectly copied symlink from host" + + run_podman rm cpcontainer +} + + +# Another symlink into host space, this one named '*' (star). cp should fail. +@test "podman cp - will not expand wildcard" { + skip_if_remote "podman-remote does not yet handle cp" + + srcdir=$PODMAN_TMPDIR/cp-test-in + dstdir=$PODMAN_TMPDIR/cp-test-out + mkdir -p $srcdir $dstdir + echo "This file lives on the host" > $srcdir/hostfile + + run_podman run --name cpcontainer $IMAGE \ + sh -c "ln -s $srcdir/hostfile /tmp/\*" + run_podman 125 cp 'cpcontainer:/tmp/*' $dstdir + + is "$output" ".*error evaluating symlinks.*lstat.*no such file or dir" \ + "Expected error from copying invalid symlink" + + # dstdir must be empty + is "$(/bin/ls -1 $dstdir)" "" "incorrectly copied symlink from host" + + run_podman rm cpcontainer +} + +############################################################################### +# cp INTO container + +# THIS IS EXTREMELY WEIRD. Podman expands symlinks in weird ways. +@test "podman cp into container: weird symlink expansion" { + skip_if_remote "podman-remote does not yet handle cp" + + srcdir=$PODMAN_TMPDIR/cp-test-in + dstdir=$PODMAN_TMPDIR/cp-test-out + mkdir -p $srcdir $dstdir + + rand_filename1=$(random_string 20) + rand_content1=$(random_string 50) + echo $rand_content1 > $srcdir/$rand_filename1 + + rand_filename2=$(random_string 20) + rand_content2=$(random_string 50) + echo $rand_content2 > $srcdir/$rand_filename2 + + rand_filename3=$(random_string 20) + rand_content3=$(random_string 50) + echo $rand_content3 > $srcdir/$rand_filename3 + + # Create tmp subdirectories in container, most with an invalid 'x' symlink + # Keep container running so we can exec into it. + run_podman run -d --name cpcontainer $IMAGE \ + sh -c "mkdir /tmp/d1;ln -s /tmp/nonesuch1 /tmp/d1/x; + mkdir /tmp/d2;ln -s /tmp/nonesuch2 /tmp/d2/x; + mkdir /tmp/d3; + trap 'exit 0' 15;while :;do sleep 0.5;done" + + # Copy file from host into container, into a file named 'x' + # Note that the second has a trailing slash; this will trigger mkdir + run_podman cp $srcdir/$rand_filename1 cpcontainer:/tmp/d1/x + is "$output" "" "output from podman cp 1" + + run_podman cp $srcdir/$rand_filename2 cpcontainer:/tmp/d2/x/ + is "$output" "" "output from podman cp 3" + + run_podman cp $srcdir/$rand_filename3 cpcontainer:/tmp/d3/x + is "$output" "" "output from podman cp 3" + + # Read back. + # In the first case, podman actually creates the file nonesuch1 (i.e. + # podman expands 'x -> nonesuch1' and, instead of overwriting x, + # creates an actual file). + run_podman exec cpcontainer cat /tmp/nonesuch1 + is "$output" "$rand_content1" "cp creates destination file" + + # In the second case, podman creates a directory nonesuch2, then + # creates a file with the same name as the input file. THIS IS WEIRD! + run_podman exec cpcontainer cat /tmp/nonesuch2/$rand_filename2 + is "$output" "$rand_content2" "cp creates destination dir and file" + + # In the third case, podman (correctly imo) creates a file named 'x' + run_podman exec cpcontainer cat /tmp/d3/x + is "$output" "$rand_content3" "cp creates file named x" + + run_podman rm -f cpcontainer + + +} + + +# rhbz1741718 : file copied into container:/var/lib/foo appears as /foo +# (docker only, never seems to have affected podman. Make sure it never does). +@test "podman cp into a subdirectory matching GraphRoot" { + skip_if_remote "podman-remote does not yet handle cp" + + # Create tempfile with random name and content + srcdir=$PODMAN_TMPDIR/cp-test-in + mkdir -p $srcdir + rand_filename=$(random_string 20) + rand_content=$(random_string 50) + echo $rand_content > $srcdir/$rand_filename + chmod 644 $srcdir/$rand_filename + + # Determine path to podman storage (eg /var/lib/c/s, or $HOME/.local/...) + run_podman info --format '{{.store.GraphRoot}}' + graphroot=$output + + # Create that directory in the container, and sleep (to keep container + # running, so we can exec into it). The trap/while is so podman-rm will + # run quickly instead of taking 10 seconds. + run_podman run -d --name cpcontainer $IMAGE sh -c \ + "mkdir -p $graphroot; trap 'exit 0' 15;while :;do sleep 0.5;done" + + # Copy from host into container. + run_podman cp $srcdir/$rand_filename cpcontainer:$graphroot/$rand_filename + + # ls, and confirm it's there. + run_podman exec cpcontainer ls -l $graphroot/$rand_filename + is "$output" "-rw-r--r-- .* 1 .* root .* 51 .* $graphroot/$rand_filename" \ + "File is copied into container in the correct (full) path" + + # Confirm it has the expected content (this is unlikely to ever fail) + run_podman exec cpcontainer cat $graphroot/$rand_filename + is "$output" "$rand_content" "Contents of file copied into container" + + run_podman rm -f cpcontainer +} + + +function teardown() { + # In case any test fails, clean up the container we left behind + run_podman rm -f cpcontainer + basic_teardown +} + +# vim: filetype=sh |