diff options
113 files changed, 2344 insertions, 1841 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index d26c1ec11..f3a0776db 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,61 +13,40 @@ env: #### #### Global variables used for all tasks #### - # File to update in home-dir with task-specific env. var values - ENVLIB: ".bash_profile" # Overrides default location (/tmp/cirrus) for repo clone + GOPATH: "/var/tmp/go" + GOSRC: "/var/tmp/go/src/github.com/containers/libpod" CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod" - # Required so $ENVLIB gets loaded + # The default is 'sh' if unspecified CIRRUS_SHELL: "/bin/bash" # Save a little typing (path relative to $CIRRUS_WORKING_DIR) SCRIPT_BASE: "./contrib/cirrus" - PACKER_BASE: "./contrib/cirrus/packer" - CIRRUS_CLONE_DEPTH: 200 # Command to prefix every output line with a timestamp # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" + # Command to log critical filesystems, types, and sizes. + DFCMD: "df -lhTx tmpfs" + CIRRUS_CLONE_DEPTH: 50 #### #### Cache-image names to test with ### - ACTIVE_CACHE_IMAGE_NAMES: >- - fedora-29-libpod-548c1c05 - fedora-28-libpod-548c1c05 - ubuntu-18-libpod-548c1c05 - rhel-7-libpod-548c1c05 - image-builder-image-1541772081 - FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-548c1c05" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-548c1c05" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-548c1c05" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-548c1c05" - # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready" - # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready" + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5170730531028992" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5170730531028992" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5170730531028992" #### #### Variables for composing new cache-images (used in PR testing) from #### base-images (pre-existing in GCE) #### + BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" # Git commits to use while building dependencies into cache-images FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" - CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" + CONMON_COMMIT: "f02c053eb37010fc76d1e2966de7f2cb9f969ef2" CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" - RUNC_COMMIT: "029124da7af7360afa781a0234d1b083550f797c" - # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) - PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7" # TODO: rhel-8,centos-7 - # Version of packer to use - PACKER_VER: "1.3.2" # Special image w/ nested-libvirt + tools for creating new cache and base images IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" - # Google-maintained base-image names - UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a" - CENTOS_BASE_IMAGE: "centos-7-v20181113" - # Manually produced base-image names (see $SCRIPT_BASE/README.md) - FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245" - PRIOR_FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-1544474897" - FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861" - # RHEL image must be imported, google bills extra for their native image. - RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" #### #### Default to NOT operating in any special-case testing mode @@ -80,8 +59,6 @@ env: #### # Freenode IRC credentials for posting status messages IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883] - # Command to register a RHEL VM to install/update packages - RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] # Needed to build GCE images, within a GCE VM SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] # User ID for cirrus to ssh into VMs @@ -89,21 +66,16 @@ env: # Name where this repositories cloud resources are located GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - # Space separated list of environment variables to unset before testing - UNSET_ENV_VARS: >- - GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX - IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE - UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT - CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE - CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI - CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA - CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH - CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST - CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME - CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME - CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION - CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX - XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER + +# Default VM to use unless set or modified by task +gce_instance: + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 2 + memory: "4Gb" + disk: 200 + # A matrix could be used here, for now just one VM + image_name: "${FEDORA_CACHE_IMAGE_NAME}" # Every *_task runs in parallel in separate VMsd. The name prefix only for reference @@ -113,6 +85,7 @@ gating_task: env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" GOSRC: "/go/src/github.com/containers/libpod" # Runs within Cirrus's "community cluster" @@ -123,12 +96,14 @@ gating_task: timeout_in: 20m + networking_script: # Don't bother going further if something is down + - 'while read host port; do nc -zv -w 13 $host $port || exit 1; done < ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt' + gate_script: # N/B: entrypoint.sh resets $GOSRC (same as make clean) - '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh lint |& ${TIMESTAMP}' - - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py |& ${TIMESTAMP}' # This task builds Podman with different buildtags to ensure the build does # not break. It also verifies all sub-commands have man pages. @@ -154,11 +129,15 @@ gating_task: # in sync at all times. vendor_task: + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + depends_on: - "gating" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" # Runs within Cirrus's "community cluster" container: @@ -171,7 +150,7 @@ vendor_task: vendor_script: - '/usr/local/bin/entrypoint.sh .install.vndr |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh vendor |& ${TIMESTAMP}' - - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh |& ${TIMESTAMP}' + - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}' @@ -181,11 +160,15 @@ vendor_task: # whether the git tree is clean. varlink_api_task: + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + depends_on: - "gating" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" # Used by tree_status.sh SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.' @@ -197,9 +180,9 @@ varlink_api_task: timeout_in: 10m - vendor_script: - - '/usr/local/bin/entrypoint.sh varlink_api_generate' - - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh' + api_md_script: + - '/usr/local/bin/entrypoint.sh varlink_api_generate |& ${TIMESTAMP}' + - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' @@ -213,7 +196,8 @@ build_each_commit_task: - "varlink_api" # $CIRRUS_BASE_BRANCH is only set when testing a PR - only_if: $CIRRUS_BRANCH != 'master' + only_if: $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' gce_instance: image_project: "libpod-218412" @@ -247,7 +231,11 @@ meta_task: env: # Space-separated list of images used by this repository state - IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}" + IMGNAMES: >- + ${FEDORA_CACHE_IMAGE_NAME} + ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + ${UBUNTU_CACHE_IMAGE_NAME} + ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df] @@ -255,6 +243,8 @@ meta_task: GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] CIRRUS_CLONE_DEPTH: 1 # source not used + timeout_in: 10m + script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}' @@ -267,38 +257,28 @@ testing_task: - "vendor" - "build_each_commit" - env: - matrix: - TEST_REMOTE_CLIENT: true - TEST_REMOTE_CLIENT: false + # Only test build cache-images, if that's what's requested + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 200 # see https://developers.google.com/compute/docs/disks#performance - # Generate multiple parallel tasks, covering all possible - # 'matrix' combinations. matrix: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" - # TODO: Make these work (also optional_testing_task below) - # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}" - # image_name: "${RHEL_CACHE_IMAGE_NAME}" - # image_name: "${CENTOS_CACHE_IMAGE_NAME}" - timeout_in: 120m - # Every *_script runs in sequence, for each task. The name prefix is for - # WebUI reference. The values may be strings... + env: + matrix: + TEST_REMOTE_CLIENT: true + TEST_REMOTE_CLIENT: false + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' @@ -306,6 +286,7 @@ testing_task: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -319,30 +300,25 @@ special_testing_task: - "vendor" - "build_each_commit" - gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 200 - # A matrix could be used here, for now just one VM - image_name: "${FEDORA_CACHE_IMAGE_NAME}" + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' env: matrix: SPECIALMODE: 'rootless' # See docs SPECIALMODE: 'in_podman' # See docs - timeout_in: 120m + timeout_in: 60m setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -351,23 +327,22 @@ special_testing_task: # necessary to execute them within a PR to validate changes. optional_testing_task: + depends_on: + - "gating" + # Only run system tests in PRs (not on merge) if magic string is present # in the PR description. Post-merge system testing is assumed to happen # later from OS distribution's build systems. only_if: >- $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' gce_instance: - image_project: "libpod-218412" matrix: image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" - # TODO: Make these work (also testing_task above) - # image_name: "${RHEL_CACHE_IMAGE_NAME}" - # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}" - # image_name: "${CENTOS_CACHE_IMAGE_NAME}" timeout_in: 60m @@ -375,13 +350,85 @@ optional_testing_task: system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' +# Test building of new cache-images for future PR testing, in this PR. +test_build_cache_images_task: + + only_if: >- + $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + + depends_on: + - "gating" + + # VMs created by packer are not cleaned up by cirrus, must allow task to complete + auto_cancellation: $CI != "true" + + gce_instance: + image_project: "libpod-218412" + zone: "us-central1-a" + cpu: 4 + memory: "4Gb" + disk: 200 + image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" + scopes: # required for image building + - compute + - devstorage.full_control + + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' + + +# Test building of new cache-images for future PR testing, in this PR. +verify_test_built_images_task: + + only_if: >- + $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + + + depends_on: + - "gating" + - "test_build_cache_images" + + gce_instance: + matrix: + # Images are generated separately, from build_images_task (below) + image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" + image_name: "fedora-29${BUILT_IMAGE_SUFFIX}" + image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" + + env: + matrix: + TEST_REMOTE_CLIENT: true + TEST_REMOTE_CLIENT: false + + # Note: A truncated form of normal testing. It only needs to confirm new images + # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME + # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' + audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' + journalctl_b_script: 'journalctl -b' + on_failure: + # Job has already failed, don't fail again and miss collecting data + failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' + failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' + failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' + + # Build new cache-images for future PR testing, but only after a PR merge. # The cache-images save install/setup time needed test every PR. The 'active' images # are selected by the 'image_name' items tasks above. Currently this requires # manually updating the names, but this could be automated (see comment below). -cache_images_task: +build_cache_images_task: # Only produce new cache-images after a PR merge, and if a magic string - # is present in the most recent commit-message. + # is present in the most recent ___commit-message___. only_if: >- $CIRRUS_BRANCH == 'master' && $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*' @@ -390,6 +437,7 @@ cache_images_task: depends_on: - "gating" - "testing" + - "rootless_testing" # VMs created by packer are not cleaned up by cirrus auto_cancellation: $CI != "true" @@ -405,6 +453,7 @@ cache_images_task: scopes: - compute - devstorage.full_control + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' @@ -430,19 +479,24 @@ success_task: depends_on: # ignores any dependent task conditions - "gating" - - "varlink_api" + - "build_each_commit" - "vendor" - - "build_each_commit_task" + - "varlink_api" - "testing" - - "rootless_testing_task" + - "special_testing" - "optional_testing" + - "test_build_cache_images" + - "verify_test_built_images" + - "build_cache_images" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" container: image: "quay.io/libpod/gate:latest" cpu: 1 memory: 1 - success_script: '$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' + success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' diff --git a/.copr/Makefile b/.copr/Makefile index 05d9eb592..71142920b 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -16,4 +16,4 @@ build_binary: clean: rm -fr rpms - rm -fr cri-o + rm -fr conmon diff --git a/.copr/prepare.sh b/.copr/prepare.sh index 1ad29da36..a40e2aadb 100644 --- a/.copr/prepare.sh +++ b/.copr/prepare.sh @@ -28,5 +28,5 @@ fi mkdir build/ git archive --prefix "libpod-${COMMIT_SHORT}/" --format "tar.gz" HEAD -o "build/libpod-${COMMIT_SHORT}.tar.gz" -git clone https://github.com/kubernetes-incubator/cri-o -cd cri-o && git checkout 4cd5a7c60349be0678d9f1b0657683324c1a2726 && git archive --prefix "crio/" --format "tar.gz" HEAD -o "../build/crio.tar.gz" +git clone https://github.com/containers/conmon +cd conmon && git checkout f02c053eb37010fc76d1e2966de7f2cb9f969ef2 && git archive --prefix "conmon/" --format "tar.gz" HEAD -o "../build/conmon.tar.gz" diff --git a/Dockerfile b/Dockerfile index 767e64570..f3afd5e25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,13 +56,13 @@ RUN set -x \ && rm -rf "$GOPATH" # Install conmon -ENV CRIO_COMMIT 7a283c391abb7bd25086a8ff91dbb36ebdd24466 +ENV CONMON_COMMIT f02c053eb37010fc76d1e2966de7f2cb9f969ef2 RUN set -x \ && export GOPATH="$(mktemp -d)" \ - && git clone https://github.com/kubernetes-sigs/cri-o.git "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ - && cd "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ + && git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \ + && cd "$GOPATH/src/github.com/containers/conmon.git" \ && git fetch origin --tags \ - && git checkout -q "$CRIO_COMMIT" \ + && git checkout -q "$CONMON_COMMIT" \ && make \ && install -D -m 755 bin/conmon /usr/libexec/podman/conmon \ && rm -rf "$GOPATH" diff --git a/Dockerfile.centos b/Dockerfile.centos index 605dc9df4..47f7182b6 100644 --- a/Dockerfile.centos +++ b/Dockerfile.centos @@ -64,15 +64,14 @@ RUN set -x \ && install -D -m 755 "$GOPATH"/bin/easyjson /usr/bin/ # Install conmon -ENV CRIO_COMMIT 7a283c391abb7bd25086a8ff91dbb36ebdd24466 +ENV CONMON_COMMIT f02c053eb37010fc76d1e2966de7f2cb9f969ef2 RUN set -x \ && export GOPATH="$(mktemp -d)" \ - && git clone https://github.com/kubernetes-sigs/cri-o.git "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ - && cd "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ + && git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \ + && cd "$GOPATH/src/github.com/containers/conmon.git" \ && git fetch origin --tags \ - && git checkout -q "$CRIO_COMMIT" \ + && git checkout -q "$CONMON_COMMIT" \ && make \ - && make bin/conmon \ && install -D -m 755 bin/conmon /usr/libexec/podman/conmon \ && rm -rf "$GOPATH" diff --git a/Dockerfile.fedora b/Dockerfile.fedora index d4bcc11ea..290fe3f82 100644 --- a/Dockerfile.fedora +++ b/Dockerfile.fedora @@ -68,15 +68,14 @@ RUN set -x \ && install -D -m 755 "$GOPATH"/bin/easyjson /usr/bin/ # Install conmon -ENV CRIO_COMMIT 7a283c391abb7bd25086a8ff91dbb36ebdd24466 +ENV CONMON_COMMIT f02c053eb37010fc76d1e2966de7f2cb9f969ef2 RUN set -x \ && export GOPATH="$(mktemp -d)" \ - && git clone https://github.com/kubernetes-sigs/cri-o.git "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ - && cd "$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" \ + && git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \ + && cd "$GOPATH/src/github.com/containers/conmon.git" \ && git fetch origin --tags \ - && git checkout -q "$CRIO_COMMIT" \ + && git checkout -q "$CONMON_COMMIT" \ && make \ - && make bin/conmon \ && install -D -m 755 bin/conmon /usr/libexec/podman/conmon \ && rm -rf "$GOPATH" @@ -259,14 +259,17 @@ changelog: ## Generate changelog $(shell cat $(TMPFILE) >> changelog.txt) $(shell rm $(TMPFILE)) -install: .gopathok install.bin install.man install.cni install.systemd ## Install binaries to system locations +install: .gopathok install.bin install.remote install.man install.cni install.systemd ## Install binaries to system locations + +install.remote: + install ${SELINUXOPT} -d -m 755 $(BINDIR) + install ${SELINUXOPT} -m 755 bin/podman-remote $(BINDIR)/podman-remote + test -z "${SELINUXOPT}" || chcon --verbose --reference=$(BINDIR)/podman bin/podman-remote install.bin: install ${SELINUXOPT} -d -m 755 $(BINDIR) install ${SELINUXOPT} -m 755 bin/podman $(BINDIR)/podman - install ${SELINUXOPT} -m 755 bin/podman-remote $(BINDIR)/podman-remote test -z "${SELINUXOPT}" || chcon --verbose --reference=$(BINDIR)/podman bin/podman - test -z "${SELINUXOPT}" || chcon --verbose --reference=$(BINDIR)/podman bin/podman-remote install.man: docs install ${SELINUXOPT} -d -m 755 $(MANDIR)/man1 @@ -7,6 +7,7 @@ approvers: - umohnani8 - giuseppe - vrothberg + - jwhonce reviewers: - mheon - baude @@ -16,3 +17,4 @@ reviewers: - umohnani8 - giuseppe - vrothberg + - jwhonce @@ -44,7 +44,7 @@ Any recent Podman release should be able to run rootless without any additional * Specializing in signing and pushing images to various storage backends. See [Skopeo](https://github.com/containers/skopeo/) for those tasks. * Container runtimes daemons for working with the Kubernetes CRI interface. - [CRI-O](https://github.com/kubernetes-sigs/cri-o) specializes in that. + [CRI-O](https://github.com/cri-o/cri-o) specializes in that. * Supporting `docker-compose`. We believe that Kubernetes is the defacto standard for composing Pods and for orchestrating containers, making Kubernetes YAML a defacto standard file format. Hence, Podman allows the @@ -67,7 +67,7 @@ The plan is to use OCI projects and best of breed libraries for different aspect - Storage: Container and image storage is managed by [containers/storage](https://github.com/containers/storage) - Networking: Networking support through use of [CNI](https://github.com/containernetworking/cni) - Builds: Builds are supported via [Buildah](https://github.com/containers/buildah). -- Conmon: [Conmon](https://github.com/kubernetes-sigs/cri-o) is a tool for monitoring OCI runtimes. It is part of the CRI-O package +- Conmon: [Conmon](https://github.com/containers/conmon) is a tool for monitoring OCI runtimes. ## Podman Information for Developers diff --git a/cmd/podman/images.go b/cmd/podman/images.go index 1c46571c3..3f755efc1 100644 --- a/cmd/podman/images.go +++ b/cmd/podman/images.go @@ -362,7 +362,7 @@ func CreateFilterFuncs(ctx context.Context, r *adapter.LocalRuntime, filters []s var filterFuncs []imagefilters.ResultFilter for _, filter := range filters { splitFilter := strings.Split(filter, "=") - if len(splitFilter) != 2 { + if len(splitFilter) < 2 { return nil, errors.Errorf("invalid filter syntax %s", filter) } switch splitFilter[0] { diff --git a/cmd/podman/info.go b/cmd/podman/info.go index a6fce7fcb..823303354 100644 --- a/cmd/podman/info.go +++ b/cmd/podman/info.go @@ -10,6 +10,7 @@ import ( "github.com/containers/libpod/pkg/adapter" "github.com/containers/libpod/version" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -60,7 +61,16 @@ func infoCmd(c *cliconfig.InfoValues) error { if err != nil { return errors.Wrapf(err, "error getting info") } + if runtime.Remote { + endpoint, err := runtime.RemoteEndpoint() + if err != nil { + logrus.Errorf("Failed to obtain server connection: %s", err.Error()) + } else { + remoteClientInfo["Connection"] = endpoint.Connection + remoteClientInfo["Connection Type"] = endpoint.Type.String() + } + remoteClientInfo["RemoteAPI Version"] = version.RemoteAPIVersion remoteClientInfo["Podman Version"] = version.Version remoteClientInfo["OS Arch"] = fmt.Sprintf("%s/%s", rt.GOOS, rt.GOARCH) diff --git a/cmd/podman/login.go b/cmd/podman/login.go index 6bf148cca..9f9631d0d 100644 --- a/cmd/podman/login.go +++ b/cmd/podman/login.go @@ -74,21 +74,6 @@ func loginCmd(c *cliconfig.LoginValues) error { sc.DockerCertPath = c.CertDir } - if c.Flag("get-login").Changed { - user, err := config.GetUserLoggedIn(sc, server) - - if err != nil { - return errors.Wrapf(err, "unable to check for login user") - } - - if user == "" { - return errors.Errorf("not logged into %s", server) - } - - fmt.Printf("%s\n", user) - return nil - } - // username of user logged in to server (if one exists) userFromAuthFile, passFromAuthFile, err := config.GetAuthentication(sc, server) // Do not return error if no credentials found in credHelpers, new credentials will be stored by config.SetAuthentication @@ -96,6 +81,14 @@ func loginCmd(c *cliconfig.LoginValues) error { return errors.Wrapf(err, "error reading auth file") } + if c.Flag("get-login").Changed { + if userFromAuthFile == "" { + return errors.Errorf("not logged into %s", server) + } + fmt.Printf("%s\n", userFromAuthFile) + return nil + } + ctx := getContext() password := c.Password diff --git a/cmd/podman/runlabel.go b/cmd/podman/runlabel.go index c426817de..e87b88992 100644 --- a/cmd/podman/runlabel.go +++ b/cmd/podman/runlabel.go @@ -152,7 +152,7 @@ func runlabelCmd(c *cliconfig.RunlabelValues) error { return err } if !c.Quiet { - fmt.Printf("command: %s\n", strings.Join(cmd, " ")) + fmt.Printf("command: %s\n", strings.Join(append([]string{os.Args[0]}, cmd[1:]...), " ")) if c.Display { return nil } diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index ea358d2d7..69d8653fe 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -99,43 +99,88 @@ contents of the ``$SPECIALMODE`` environment variable. then execute `make localsystem` from the repository root. -### ``cache_images`` Task +### ``test_build_cache_images_task`` Task -Modifying the contents of cache-images is done by making changes to -one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Testing -those changes currently requires adding a temporary commit to a PR that -updates ``.cirrus.yml``: - -* Remove all task sections except ``cache_images_task``. -* Remove the ``only_if`` condition and ``depends_on`` dependencies - -The new image names will be displayed at the end of output, assuming the build -is successful, at that point the temporary commit may be removed. Finally, -the new names may be used as ``image_name`` values in ``.cirrus.yml``. +Modifying the contents of cache-images is tested by making changes to +one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then +in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***`` ***N/B: Steps below are performed by automation*** -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another - round of the ``gating`` and ``testing`` tasks (above). - -2. Assuming tests pass, if the commit message contains the magic string - ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise - simply mark the master branch as 'passed'. +1. ``setup_environment.sh``: Same as for other tasks. -3. ``setup_environment.sh``: Same as for other tasks. - -4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) +2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) to produce new VM images. Create a new VM from each base-image, connect to them with ``ssh``, and perform the steps as defined by the - ``$PACKER_BASE/libpod_images.json`` file: + ``$PACKER_BASE/libpod_images.yml`` file: 1. On a base-image VM, as root, copy the current state of the repository into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``integration_testing`` task (above). For example, - ``fedora_setup.sh``. - 3. If successful, shut down each VM and create a new GCE Image - named with the base image, and the commit sha of the merge. + use. For example, ``fedora_setup.sh``. + 3. If successful, shut down each VM and record the names, and dates + into a json manifest file. + 4. Move the manifest file, into a google storage bucket object. + This is a retained as a secondary method for tracking/auditing + creation of VM images, should it ever be needed. + +### ``verify_test_built_images`` Task + +Only runs following successful ``test_build_cache_images_task`` task. Uses +images following the standard naming format; ***however, only runs a limited +sub-set of automated tests***. Validating newly built images fully, requires +updating ``.cirrus.yml``. + +***Manual Steps:*** Assuming `verify_test_built_images` passes, then +you'll find the new image names displayed at the end of the +`test_build_cache_images_task` in the `build_vm_images` output. +For example: + + +``` +...cut... +==> Builds finished. The artifacts of successful builds are: +--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 +--> ubuntu-18: +--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 +--> fedora-29: +--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 +``` + +Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the +images from above: + + +```yaml +env: + ...cut... + #### + #### Cache-image names to test with + ### + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" + ...cut... +``` + +***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`, +take care to also *update the PR description* to remove +the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and +`--force` pushing would needlessly cause Cirrus-CI to build +and test images again. + + +### ``build_cache_images`` Task *(Deprecated)* + +Exactly the same as ``test_build_cache_images_task`` task, but only runs on +the master branch. Requires a magic string to be in the `HEAD` +commit message: ``***CIRRUS: BUILD IMAGES***`` + +When successful, the manifest file along with all VM disks, are moved +into a dedicated google storage bucket, separate from the one used by +`test_build_cache_images_task`. These may be used to create new cache-images for +PR testing by manually importing them as described above. + ### Base-images @@ -170,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required: or [end-user credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) -* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths - to a `rhel-server-ec2-*.raw.xz` and it's cooresponding - checksum file. These must be supplied manually because - they're not available directly via URL like other images. - -* ``RHSM_COMMAND`` contains the complete string needed to register - the VM for installing package dependencies. The VM will be de-registered - upon completion. - * Optionally, CSV's may be specified to ``PACKER_BUILDS`` to limit the base-images produced. For example, ``PACKER_BUILDS=fedora,image-builder-image``. @@ -224,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images: $ cd libpod/contrib/cirrus/packer $ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \ - RHEL_IMAGE_FILE=<VALUE> \ - RHEL_CSUM_FILE=<VALUE> \ - RHSM_COMMAND=<VALUE> \ PACKER_BUILDS=<OPTIONAL> ``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 43eb3c057..805aba428 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,10 @@ set -e source $(dirname $0)/lib.sh -req_env_var CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE +ENV_VARS='CNI_COMMIT CONMON_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +req_env_var $ENV_VARS +# Must also be made available through make, into packer process +export $ENV_VARS show_env_vars @@ -46,7 +49,4 @@ make libpod_images \ URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json" gsutil cp packer-manifest.json "$URI" -echo "Finished." -echo "Any tarball URI's referenced above at at $URI" -echo "may be used to create VM images suitable for use in" -echo ".cirrus.yml as values for the 'image_name' keys." +echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index c7d381318..e7f582b42 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -9,7 +9,6 @@ cd "$GOSRC" if [[ "$SPECIALMODE" == "in_podman" ]] then - set -x ${CONTAINER_RUNTIME} run --rm --privileged --net=host \ -v $GOSRC:$GOSRC:Z \ --workdir $GOSRC \ @@ -26,30 +25,19 @@ then elif [[ "$SPECIALMODE" == "rootless" ]] then req_env_var ROOTLESS_USER - set -x - ssh $ROOTLESS_USER@localhost \ + + if [[ "$USER" == "$ROOTLESS_USER" ]] + then + $GOSRC/$SCRIPT_BASE/rootless_test.sh + else + ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ $GOSRC/$SCRIPT_BASE/rootless_test.sh - exit $? + fi else - set -x make make install PREFIX=/usr ETCDIR=/etc make test-binaries - make install.tools - clean_env - - case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;; - fedora-29) ;& # Continue to the next item - fedora-28) ;& - centos-7) ;& - rhel-7) - make podman-remote - install bin/podman-remote /usr/bin - ;; - *) bad_os_id_ver ;; - esac if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remoteintegration diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index f422fe935..334202aa9 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,35 +3,92 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. +# Global details persist here +source /etc/environment # not always loaded under all circumstances + # Under some contexts these values are not set, make sure they are. -export USER="$(whoami)" -export HOME="$(getent passwd $USER | cut -d : -f 6)" - -# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh -# Pick some reasonable defaults -ENVLIB=${ENVLIB:-.bash_profile} -CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}" -GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +USER="$(whoami)" +HOME="$(getent passwd $USER | cut -d : -f 6)" +[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3) +GID=$(getent passwd $USER | cut -d : -f 4) + +# Essential default paths, many are overriden when executing under Cirrus-CI +export GOPATH="${GOPATH:-/var/tmp/go}" +if type -P go &> /dev/null +then + # required for go 1.12+ + export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}" + eval "$(go env)" + # required by make and other tools + export $(go env | cut -d '=' -f 1) + + # Ensure compiled tooling is reachable + export PATH="$PATH:$GOPATH/bin" +fi +CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}" +export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" +export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}" +SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" +# Saves typing / in case location ever moves SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus} PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} -CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF} # a human -CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD} -CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD} -SPECIALMODE="${SPECIALMODE:-none}" -export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} -if ! [[ "$PATH" =~ "/usr/local/bin" ]] +cd $GOSRC +if type -P git &> /dev/null then - export PATH="$PATH:/usr/local/bin" + CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} +else # pick something unique and obviously not from Cirrus + CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)} fi -# In ci/testing environment, ensure variables are always loaded -if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]] +# Defaults when not running under CI +export CI="${CI:-false}" +CIRRUS_CI="${CIRRUS_CI:-false}" +CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" +CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} +CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover +CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique +# Vars. for image-building +PACKER_VER="1.3.5" +# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) + +# Base-images rarely change, define them here so they're out of the way. +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" +# Google-maintained base-image names +UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" +# Manually produced base-image names (see $SCRIPT_BASE/README.md) +FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664" +PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" +# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" + +# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) +ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' +# Unsafe env. vars for display +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' + +SPECIALMODE="${SPECIALMODE:-none}" +TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" +export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} + +# When running as root, this may be empty or not, as a user, it MUST be set. +if [[ "$USER" == "root" ]] then - # Make sure this is always loaded - source "$HOME/$ENVLIB" + ROOTLESS_USER="${ROOTLESS_USER:-}" +else + ROOTLESS_USER="${ROOTLESS_USER:-$USER}" fi +# GCE image-name compatible string representation of distribution name +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" +# Combined to ease soe usage +OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" + # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty req_env_var() { @@ -57,81 +114,30 @@ req_env_var() { done } -# Some env. vars may contain secrets. Display values for known "safe" -# and useful variables. -# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables show_env_vars() { - # This is almost always multi-line, print it separately - echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE" - echo " -BUILDTAGS $BUILDTAGS -BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX -ROOTLESS_USER $ROOTLESS_USER -CI $CI -CIRRUS_CI $CIRRUS_CI -CI_NODE_INDEX $CI_NODE_INDEX -CI_NODE_TOTAL $CI_NODE_TOTAL -CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION -CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH -CIRRUS_BASE_SHA $CIRRUS_BASE_SHA -CIRRUS_BRANCH $CIRRUS_BRANCH -CIRRUS_BUILD_ID $CIRRUS_BUILD_ID -CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO -CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH -CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH -CIRRUS_PR $CIRRUS_PR -CIRRUS_TAG $CIRRUS_TAG -CIRRUS_OS $CIRRUS_OS -OS $OS -CIRRUS_TASK_NAME $CIRRUS_TASK_NAME -CIRRUS_TASK_ID $CIRRUS_TASK_ID -CIRRUS_REPO_NAME $CIRRUS_REPO_NAME -CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER -CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME -CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL -CIRRUS_SHELL $CIRRUS_SHELL -CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR -CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION -CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR -CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST -SPECIALMODE $SPECIALMODE -$(go env) -PACKER_BUILDS $PACKER_BUILDS - " | while read NAME VALUE + echo "Showing selection of environment variable definitions:" + _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ + egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ + sort -u) + for _env_var_name in $_ENV_VAR_NAMES do - [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\"" + # Supports older BASH versions + printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)" done echo "" echo "##### $(go version) #####" echo "" } -# Unset environment variables not needed for testing purposes -clean_env() { - req_env_var UNSET_ENV_VARS - echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables" - unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only -} - die() { - echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" + echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" exit ${1:-1} } -# Return a GCE image-name compatible string representation of distribution name -os_release_id() { - eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")" - echo "$ID" -} - -# Return a GCE image-name compatible string representation of distribution major version -os_release_ver() { - eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")" - echo "$VERSION_ID" | cut -d '.' -f 1 -} - bad_os_id_ver() { - echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" + echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" exit 42 } @@ -140,8 +146,8 @@ stub() { } ircmsg() { - req_env_var CIRRUS_TASK_ID - [[ -n "$*" ]] || die 9 "ircmsg() invoked without args" + req_env_var CIRRUS_TASK_ID IRCID + [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" NICK="podbot_$CIRRUS_TASK_ID" @@ -153,8 +159,9 @@ ircmsg() { } setup_rootless() { - req_env_var ROOTLESS_USER GOSRC ENVLIB + req_env_var ROOTLESS_USER GOSRC + # Only do this once if passwd --status $ROOTLESS_USER then echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code" @@ -162,12 +169,7 @@ setup_rootless() { return 0 fi - # Only do this once cd $GOSRC - make install.catatonit - go get github.com/onsi/ginkgo/ginkgo - go get github.com/onsi/gomega/... - # Guarantee independence from specific values ROOTLESS_UID=$[RANDOM+1000] ROOTLESS_GID=$[RANDOM+1000] @@ -177,7 +179,8 @@ setup_rootless() { chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC" echo "creating ssh keypair for $USER" - ssh-keygen -P "" -f $HOME/.ssh/id_rsa + [[ -r "$HOME/.ssh/id_rsa" ]] || \ + ssh-keygen -P "" -f "$HOME/.ssh/id_rsa" echo "Allowing ssh key for $ROOTLESS_USER" (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh") @@ -192,16 +195,19 @@ setup_rootless() { echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \ tee -a /etc/subuid >> /etc/subgid - echo "Copying $HOME/$ENVLIB" - install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \ - "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB" - - echo "Configuring user's go environment variables" - su --login --command 'go env' $ROOTLESS_USER | \ - while read envline - do - X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X" - done + # Env. vars set by Cirrus and setup_environment.sh must be explicitly + # transfered to the test-user. + echo "Configuring rootless user's environment variables:" + echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()" + _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ + egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ + egrep "$ROOTLESS_ENV_RE" | \ + sort -u) + for _env_var_name in $_ENV_VAR_NAMES + do + # Works with older versions of bash + printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc" + done } # Helper/wrapper script to only show stderr/stdout on non-zero exit @@ -239,6 +245,7 @@ install_cni_plugins() { } install_runc_from_git(){ + req_env_var GOPATH OS_RELEASE_ID RUNC_COMMIT wd=$(pwd) DEST="$GOPATH/src/github.com/opencontainers/runc" rm -rf "$DEST" @@ -246,13 +253,17 @@ install_runc_from_git(){ cd "$DEST" ooe.sh git fetch origin --tags ooe.sh git checkout -q "$RUNC_COMMIT" - ooe.sh make static BUILDTAGS="seccomp apparmor selinux" + if [[ "${OS_RELEASE_ID}" == "ubuntu" ]] + then + ooe.sh make static BUILDTAGS="seccomp apparmor" + else + ooe.sh make BUILDTAGS="seccomp selinux" + fi sudo install -m 755 runc /usr/bin/runc cd $wd } install_runc(){ - OS_RELEASE_ID=$(os_release_id) echo "Installing RunC from commit $RUNC_COMMIT" echo "Platform is $OS_RELEASE_ID" req_env_var GOPATH RUNC_COMMIT OS_RELEASE_ID @@ -283,23 +294,21 @@ install_buildah() { ooe.sh sudo make install } -# Requires $GOPATH and $CRIO_COMMIT to be set +# Requires $GOPATH and $CONMON_COMMIT to be set install_conmon(){ - echo "Installing conmon from commit $CRIO_COMMIT" - req_env_var GOPATH CRIO_COMMIT - DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" + echo "Installing conmon from commit $CONMON_COMMIT" + req_env_var GOPATH CONMON_COMMIT + DEST="$GOPATH/src/github.com/containers/conmon.git" rm -rf "$DEST" - ooe.sh git clone https://github.com/kubernetes-sigs/cri-o.git "$DEST" + ooe.sh git clone https://github.com/containers/conmon.git "$DEST" cd "$DEST" ooe.sh git fetch origin --tags - ooe.sh git checkout -q "$CRIO_COMMIT" + ooe.sh git checkout -q "$CONMON_COMMIT" ooe.sh make sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon } install_criu(){ - OS_RELEASE_ID=$(os_release_id) - OS_RELEASE_VER=$(os_release_ver) echo "Installing CRIU" echo "Installing CRIU from commit $CRIU_COMMIT" echo "Platform is $OS_RELEASE_ID" @@ -309,21 +318,6 @@ install_criu(){ ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa ooe.sh sudo -E apt-get -qq -y update ooe.sh sudo -E apt-get -qq -y install criu - elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then - echo "Configuring Repositories for latest CRIU" - ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF -[adrian-criu-el7] -name=Copr repo for criu-el7 owned by adrian -baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg -repo_gpgcheck=0 -enabled=1 -enabled_metadata=1 -EOF - ooe.sh sudo yum -y install criu elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then echo "Using CRIU from distribution" else @@ -338,16 +332,6 @@ EOF fi } -install_packer_copied_files(){ - # Install cni config, policy and registry config - sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \ - /etc/cni/net.d/87-podman-bridge.conflist - sudo install -D -m 755 /tmp/libpod/test/policy.json \ - /etc/containers/policy.json - sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \ - /etc/containers/registries.d/registry.access.redhat.com.yaml -} - install_varlink() { echo "Installing varlink from the cheese-factory" ooe.sh sudo -H pip3 install varlink @@ -376,7 +360,7 @@ rh_finalize(){ fi echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") - [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic + sudo $PKG clean all sudo rm -rf /var/cache/{yum,dnf} sudo rm -f /etc/udev/rules.d/*-persistent-*.rules sudo touch /.unconfigured # force firstboot to run @@ -389,25 +373,3 @@ ubuntu_finalize(){ sudo rm -rf /var/cache/apt _finalize } - -rhel_exit_handler() { - set +ex - req_env_var GOPATH RHSMCMD - cd / - sudo rm -rf "$RHSMCMD" - sudo rm -rf "$GOPATH" - sudo subscription-manager remove --all - sudo subscription-manager unregister - sudo subscription-manager clean -} - -rhsm_enable() { - req_env_var RHSM_COMMAND - export GOPATH="$(mktemp -d)" - export RHSMCMD="$(mktemp)" - trap "rhel_exit_handler" EXIT - # Avoid logging sensitive details - echo "$RHSM_COMMAND" > "$RHSMCMD" - ooe.sh sudo bash "$RHSMCMD" - sudo rm -rf "$RHSMCMD" -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index ce51f8ad2..1f05b3bb5 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -10,12 +10,13 @@ rc=0 function check_result { testnum=$(expr $testnum + 1) - if [ "$1" = "$2" ]; then - echo "ok $testnum $3 = $1" + MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//') + if [ "$MSG" = "$2" ]; then + echo "ok $testnum $3 = $MSG" else echo "not ok $testnum $3" echo "# expected: $2" - echo "# actual: $1" + echo "# actual: $MSG" rc=1 fi } @@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments' unset FOO BAR test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty' test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty' - -# OK if desired envariable is unset +# OK if desired envariable was unset FOO=1 test_rev FOO 0 '' +# OK if multiple vars are non-empty +FOO="stuff" +BAR="things" +ENV_VARS="FOO BAR" +test_rev "$ENV_VARS" 0 '' +unset BAR + # ...but error if any single desired one is unset test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty' diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_master_failure.sh index 4b09331d3..1fc15cdf9 100755 --- a/contrib/cirrus/notice_master_failure.sh +++ b/contrib/cirrus/notice_master_failure.sh @@ -12,7 +12,7 @@ NOR="$(echo -n -e '\x0f')" if [[ "$CIRRUS_BRANCH" =~ "master" ]] then BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID" - ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL. Please investigate, and re-run if appropriate." + ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(OS_RELEASE_ID)-$(OS_RELEASE_VER): $BURL. Please investigate, and re-run if appropriate." fi # This script assumed to be executed on failure diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 91a1dfeef..d03d22abe 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.2 +PACKER_VER ?= 1.3.5 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -49,6 +49,7 @@ ifndef PACKER_BUILDS $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif ./packer build -only=${PACKER_BUILDS} \ + -force \ -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ @@ -78,16 +79,11 @@ endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) endif -ifndef RHEL_IMAGE_FILE - $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) -endif -ifndef RHEL_CSUM_FILE - $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) -endif -ifndef RHSM_COMMAND - $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) +ifndef PACKER_BUILDS + $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif PACKER_CACHE_DIR=/tmp ./packer build \ + -force \ -var TIMESTAMP=$(TIMESTAMP) \ -var TTYDEV=$(TTYDEV) \ -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ @@ -95,10 +91,5 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ - -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ - -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ - -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ - -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh deleted file mode 100644 index 91b1963c2..000000000 --- a/contrib/cirrus/packer/centos_setup.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT - -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install centos-release-scl epel-release - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - runc \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_cni_plugins - -install_buildah - -install_conmon - -install_criu - -install_packer_copied_files - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh deleted file mode 100644 index 606c4f336..000000000 --- a/contrib/cirrus/packer/fah_base-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ - -# N/B: This script is not intended to be run by humans. It is used to configure the -# FAH base image for importing, so that it will boot in GCE. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -install_ooe - -if [[ "$1" == "pre" ]] -then - echo "Upgrading Atomic Host" - setenforce 0 - ooe.sh atomic host upgrade - - echo "Configuring Repositories" - ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF -[ngompa-gce-oslogin] -name=Copr repo for gce-oslogin owned by ngompa -baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg -repo_gpgcheck=0 -enabled=1 -enabled_metadata=1 -EOF - echo "Installing necessary packages and google services" - # Google services are enabled by default, upon install. - ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin - echo "Rebooting..." - systemctl reboot # Required for upgrade + package installs to be active -elif [[ "$1" == "post" ]] -then - echo "Enabling necessary services" - systemctl enable rngd # Must reboot before enabling - rh_finalize - echo "SUCCESS!" -else - echo "Expected to be called with 'pre' or 'post'" - exit 6 -fi diff --git a/contrib/cirrus/packer/fah_setup.sh b/contrib/cirrus/packer/fah_setup.sh deleted file mode 100644 index 18c4db0af..000000000 --- a/contrib/cirrus/packer/fah_setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject fah VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE - -install_ooe - -ooe.sh sudo atomic host upgrade - -ooe.sh sudo rpm-ostree uninstall cloud-init - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index c0a1e422c..2e6d3eceb 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -15,9 +15,8 @@ install_ooe echo "Updating packages" ooe.sh dnf -y update -echo "Installing necessary packages and google services" -ooe.sh dnf -y copr enable ngompa/gce-oslogin -ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin +echo "Installing necessary packages and google services" +ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin echo "Enabling services" ooe.sh systemctl enable rngd diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 36a65eb71..e031129d7 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT +req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CONMON_COMMIT CRIU_COMMIT install_ooe @@ -22,6 +22,7 @@ ooe.sh sudo dnf install -y \ bats \ btrfs-progs-devel \ bzip2 \ + criu \ device-mapper-devel \ emacs-nox \ findutils \ @@ -35,6 +36,7 @@ ooe.sh sudo dnf install -y \ gpgme-devel \ iptables \ iproute \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -68,16 +70,14 @@ ooe.sh sudo dnf install -y \ install_varlink +install_conmon + CNI_COMMIT=$FEDORA_CNI_COMMIT install_cni_plugins install_buildah -install_conmon - -install_criu - -install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh rh_finalize # N/B: Halts system! diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index bf568b40e..560cb321c 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -11,24 +11,16 @@ variables: TIMESTAMP: # Required for output from qemu builders TTYDEV: - # RHEL images require click-through agreements to obtain (required) - RHEL_BASE_IMAGE_NAME: - RHEL_IMAGE_FILE: - RHEL_CSUM_FILE: - # RHEL requires a subscription to install/update packages - RHSM_COMMAND: # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2' + # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE - FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2" - FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM" - FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE + PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" + PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" + PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' @@ -45,28 +37,9 @@ variables: sensitive-variables: - 'GOOGLE_APPLICATION_CREDENTIALS' - 'GCP_PROJECT_ID' - - 'RHSM_COMMAND' # What images to produce in which cloud builders: - - name: '{{user `IBI_BASE_NAME`}}' - type: 'googlecompute' - image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}' - image_family: '{{user `IBI_BASE_NAME`}}' - source_image_project_id: 'centos-cloud' - source_image_family: 'centos-7' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - communicator: 'ssh' - ssh_username: 'centos' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - # Enable nested virtualization in case it's ever needed - image_licenses: - - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx' - min_cpu_platform: "Intel Broadwell" # nested-virt requirement - - &nested_virt name: 'fedora' type: 'qemu' @@ -109,18 +82,6 @@ builders: iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - <<: *nested_virt - name: 'fah' - iso_url: '{{user `FAH_IMAGE_URL`}}' - iso_checksum_url: '{{user `FAH_CSUM_URL`}}' - disk_size: 10240 - - - <<: *nested_virt - name: 'rhel' - iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}' - iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}' - disk_size: 10240 - provisioners: - type: 'shell' inline: @@ -150,7 +111,6 @@ provisioners: - 'GOSRC=/tmp/libpod' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' - <<: *shell_script inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] @@ -161,7 +121,7 @@ provisioners: post-processors: - - type: "compress" - only: ['fedora', 'prior-fedora', 'fah', 'rhel'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -180,14 +140,4 @@ post-processors: image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['fah'] - image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FAH_IMAGE_URL`}}' - image_family: '{{user `FAH_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['rhel'] - image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}' - image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}' - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 30ad0723a..34d4db7fb 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -3,17 +3,14 @@ # All of these are required variables: # Names of GCE Base images to start from, in .cirrus.yml - RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}' - CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}' UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}' # libpod dependencies to build and install into images FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}" CNI_COMMIT: "{{env `CNI_COMMIT`}}" - CRIO_COMMIT: "{{env `CRIO_COMMIT`}}" + CONMON_COMMIT: "{{env `CONMON_COMMIT`}}" CRIU_COMMIT: "{{env `CRIU_COMMIT`}}" RUNC_COMMIT: "{{env `RUNC_COMMIT`}}" @@ -25,7 +22,6 @@ variables: # Protected credentials, decrypted by Cirrus at runtime GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' - RHSM_COMMAND: '{{env `RHSM_COMMAND`}}' SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' @@ -37,7 +33,6 @@ variables: sensitive-variables: - 'GCE_SSH_USERNAME' - 'GCP_PROJECT_ID' - - 'RHSM_COMMAND' - 'SERVICE_ACCOUNT' # What images to produce in which cloud @@ -60,14 +55,6 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image - name: 'rhel-7' - source_image: '{{user `RHEL_BASE_IMAGE`}}' - - - <<: *gce_hosted_image - name: 'centos-7' - source_image: '{{user `CENTOS_BASE_IMAGE`}}' - - - <<: *gce_hosted_image name: 'fedora-29' source_image: '{{user `FEDORA_BASE_IMAGE`}}' @@ -75,10 +62,6 @@ builders: name: 'fedora-28' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' - - <<: *gce_hosted_image - name: 'fah-29' - source_image: '{{user `FAH_BASE_IMAGE`}}' - # The brains of the operation, making actual modifications to the base-image. provisioners: - type: 'file' @@ -91,15 +74,10 @@ provisioners: - 'GOSRC=/tmp/libpod' - 'CNI_COMMIT={{user `CNI_COMMIT`}}' - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}' - - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}' + - 'CONMON_COMMIT={{user `CONMON_COMMIT`}}' - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}' - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' post-processors: - # Store VM disk in GCP storage, where it will expire based on a defined - # lifecycle. This prevents GCE from filling with disused images. - - - type: 'googlecompute-export' - paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz'] - - type: 'manifest' # writes packer-manifest.json + - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/rhel_base-setup.sh b/contrib/cirrus/packer/rhel_base-setup.sh deleted file mode 100644 index 8d5892d7d..000000000 --- a/contrib/cirrus/packer/rhel_base-setup.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# rhel base image for importing, so that it will boot in GCE - -set -e - -[[ "$1" == "post" ]] || exit 0 # pre stage is not needed - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var RHSM_COMMAND - -install_ooe - -rhsm_enable - -echo "Setting up repos" -# Frequently needed -ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - -# Required for google to manage ssh keys -ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM -[google-cloud-compute] -name=google-cloud-compute -baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOM - -echo "Updating all packages" -ooe.sh sudo yum -y update - -echo "Installing/removing packages" -ooe.sh sudo yum -y install rng-tools google-compute-engine google-compute-engine-oslogin - -echo "Enabling critical services" -ooe.sh sudo systemctl enable \ - rngd \ - google-accounts-daemon \ - google-clock-skew-daemon \ - google-instance-setup \ - google-network-daemon \ - google-shutdown-scripts \ - google-startup-scripts - -rhel_exit_handler # release subscription! - -echo "Configuring boot" -cat << "EOF" | sudo tee /etc/default/grub -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)" -GRUB_DEFAULT=saved -GRUB_DISABLE_SUBMENU=true -GRUB_TERMINAL="serial console" -GRUB_SERIAL_COMMAND="serial --speed=38400" -GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,38400n8" -GRUB_DISABLE_RECOVERY="true" -EOF -sudo grub2-mkconfig -o /boot/grub2/grub.cfg - -echo "Configuring networking" -ooe.sh sudo nmcli connection modify 'System eth0' 802-3-ethernet.mtu 1460 -ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect yes -ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect-priority -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.method auto -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-send-hostname yes -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-timeout 0 -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.never-default no -ooe.sh /usr/bin/google_instance_setup - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh deleted file mode 100644 index 45f5c3e9b..000000000 --- a/contrib/cirrus/packer/rhel_setup.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND - -install_ooe - -rhsm_enable - -ooe.sh sudo yum -y erase "rh-amazon-rhui-client*" -ooe.sh sudo subscription-manager repos "--disable=*" -ooe.sh sudo subscription-manager repos \ - --enable=rhel-7-server-rpms \ - --enable=rhel-7-server-optional-rpms \ - --enable=rhel-7-server-extras-rpms \ - --enable=rhel-server-rhscl-7-rpms - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python2-pyyaml \ - python34-dateutil \ - python34-psutil \ - python34-pytoml \ - python34-PyYAML \ - runc \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_cni_plugins - -install_buildah - -install_conmon - -install_criu - -install_packer_copied_files - -rhel_exit_handler # release subscription! - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index d3ac8bddb..56d7f962e 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT +req_env_var SCRIPT_BASE CNI_COMMIT CONMON_COMMIT CRIU_COMMIT install_ooe @@ -26,6 +26,8 @@ ooe.sh sudo -E apt-get -qq install software-properties-common # Required to have Go 1.11 on Ubuntu 18.0.4 ooe.sh sudo -E add-apt-repository --yes ppa:longsleep/golang-backports +ooe.sh sudo -E add-apt-repository --yes ppa:projectatomic/ppa +ooe.sh sudo -E add-apt-repository --yes ppa:criu/ppa ooe.sh sudo -E apt-get -qq update || sudo -E apt-get -qq update ooe.sh sudo -E apt-get -qq install \ @@ -36,6 +38,8 @@ ooe.sh sudo -E apt-get -qq install \ bison \ btrfs-tools \ build-essential \ + cri-o-runc \ + criu \ curl \ e2fslibs-dev \ emacs-nox \ @@ -45,6 +49,7 @@ ooe.sh sudo -E apt-get -qq install \ golang \ iproute2 \ iptables \ + jq \ libaio-dev \ libapparmor-dev \ libcap-dev \ @@ -89,20 +94,17 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub ooe.sh sudo update-grub -install_runc - install_conmon -install_criu - install_cni_plugins install_buildah -install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh install_varlink +sudo mkdir -p /etc/containers sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\ -o /etc/containers/registries.conf diff --git a/contrib/cirrus/required_host_ports.txt b/contrib/cirrus/required_host_ports.txt new file mode 100644 index 000000000..9248e497a --- /dev/null +++ b/contrib/cirrus/required_host_ports.txt @@ -0,0 +1,4 @@ +github.com 22 +docker.io 443 +quay.io 443 +registry.fedoraproject.org 443 diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index eab06bac0..3b668034b 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -1,22 +1,26 @@ #!/bin/bash set -e -source $HOME/.bash_profile -cd $GOSRC source $(dirname $0)/lib.sh -req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER - if [[ "$UID" == "0" ]] then echo "Error: Expected to be running as a regular user" exit 1 fi +# Ensure environment setup correctly +req_env_var GOSRC ROOTLESS_USER + echo "." echo "Hello, my name is $USER and I live in $PWD can I be your friend?" +echo "." + +export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u) +show_env_vars +set -x cd "$GOSRC" make make varlink_generate diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 4dbd56ed9..f40405e8d 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -4,11 +4,18 @@ set -e source $(dirname $0)/lib.sh -req_env_var USER HOME ENVLIB SCRIPT_BASE CIRRUS_BUILD_ID +req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH -[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash - -cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions +# Ensure this script only executes successfully once and always logs ending timestamp +[[ ! -e "$SETUP_MARKER_FILEPATH" ]] || exit 0 +exithandler() { + RET=$? + set +e + show_env_vars + echo "$(basename $0) exit status: $RET" + [[ "$RET" -eq "0" ]] && date +%s >> "SETUP_MARKER_FILEPATH" +} +trap exithandler EXIT # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git @@ -19,71 +26,70 @@ do fi done -# Setup env. vars common to all tasks/scripts/platforms and -# ensure they return for every following script execution. -MARK="# Added by $0, manual changes will be lost." -touch "$HOME/$ENVLIB" -if ! grep -q "$MARK" "$HOME/$ENVLIB" -then - cp "$HOME/$ENVLIB" "$HOME/${ENVLIB}_original" - # N/B: Single-quote items evaluated every time, double-quotes only once (right now). - for envstr in \ - "$MARK" \ - "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \ - "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \ - "export TRAVIS=\"1\"" \ - "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \ - "export OS_RELEASE_ID=\"$(os_release_id)\"" \ - "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ - "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \ - "export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \ - "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \ - "export GOPATH=\"/var/tmp/go\"" \ - 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ - 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' - do - # Make permanent in later shells, and set in current shell - X=$(echo "$envstr" | tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - done +# Sometimes environment setup needs to vary between distros +# Note: This should only be used for environment variables, and temporary workarounds. +# Anything externally dependent, should be made fixed-in-time by adding to +# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images +# (see docs). +case "${OS_REL_VER}" in + ubuntu-18) ;; + fedora-29) + # Occasionally, and seemingly only on F29 the root disk fails to expand + # upon boot. When this happens, any number of failures could occur if + # space runs out. Until there is time to investigate the actual cause, + # workaround this problem by detecting it and acting accordingly. + REMAINING=$(df /dev/sda1 | tail -1 | awk '{print $4}') + if [[ "$REMAINING" -lt "100000000" ]] # .cirrus.yml specifies 200gig + then + echo "Fixing failure to expand root filesystem" + growpart /dev/sda 1 # device guaranteed by cloud provider + resize2fs /dev/sda1 # growpart & resuze guaranteed by base-image + fi + ;; + fedora-28) ;; + centos-7) # Current VM is an image-builder-image no local podman/testing + echo "No further setup required for VM image building" + exit 0 + ;; + *) bad_os_id_ver ;; +esac - # Some setup needs to vary between distros - case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) - # Always install runc on Ubuntu - install_runc_from_git - ;; - fedora-29) - CON_SEL="https://kojipkgs.fedoraproject.org/packages/container-selinux/2.100/1.git3b78187.fc29/noarch/container-selinux-2.100-1.git3b78187.fc29.noarch.rpm" - echo ">>>>> OVERRIDING container-selinux WITH $CON_SEL <<<<<" - dnf -y install $CON_SEL - echo ">>>>> OVERRIDING criu and selinux-policy with latest package <<<<<" - dnf -y upgrade criu selinux-policy - ;& # Continue to the next item - fedora-28) - echo ">>>>> OVERRIDING source-built runc with latest package <<<<<" - dnf update -y runc - ;& # Continue to the next item - centos-7) ;& - rhel-7) - ;; - *) bad_os_id_ver ;; - esac +cd "${GOSRC}/" +# Reload to incorporate any changes from above +source "$SCRIPT_BASE/lib.sh" - cd "${GOSRC}/" - # Reload to incorporate any changes from above - source "$SCRIPT_BASE/lib.sh" +echo "Installing cni config, policy and registry config" +req_env_var GOSRC +sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \ + /etc/cni/net.d/87-podman-bridge.conflist +sudo install -D -m 755 $GOSRC/test/policy.json \ + /etc/containers/policy.json +sudo install -D -m 755 $GOSRC/test/registries.conf \ + /etc/containers/registries.conf +# cri-o if installed will mess with testing in non-obvious ways +rm -f /etc/cni/net.d/*cri* - case "$SPECIALMODE" in - rootless) - X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - setup_rootless - ;; - in_podman) # Assumed to be Fedora - dnf install -y podman buildah - $SCRIPT_BASE/setup_container_environment.sh - ;; - esac -fi +make install.tools -show_env_vars +case "$SPECIALMODE" in + none) ;; # Do the normal thing + rootless) + # Only do this once, even if ROOTLESS_USER (somehow) changes + if ! grep -q 'ROOTLESS_USER' /etc/environment + then + X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + setup_rootless + fi + ;; + in_podman) # Assumed to be Fedora + dnf install -y podman buildah + $SCRIPT_BASE/setup_container_environment.sh + ;; + *) + die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" +esac diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index dd5ef511d..a2cc1af05 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -5,17 +5,12 @@ source $(dirname $0)/lib.sh req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER -clean_env - set -x cd "$GOSRC" -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;& # Continue to the next item - fedora-28) ;& - fedora-29) ;& - centos-7) ;& - rhel-7) +case "${OS_RELEASE_ID}" in + ubuntu) ;& # Continue to the next item + fedora) make install.tools make make test-binaries diff --git a/contrib/cirrus/test/test_dot_cirrus_yaml.py b/contrib/cirrus/test/test_dot_cirrus_yaml.py deleted file mode 100755 index 2894bc45e..000000000 --- a/contrib/cirrus/test/test_dot_cirrus_yaml.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/env python3 - -import sys -import os -import os.path -import unittest -import warnings -import yaml - -class TestCaseBase(unittest.TestCase): - - SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0]))) - CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR', - '{0}/../../../'.format(SCRIPT_PATH)) - - def setUp(self): - os.chdir(self.CIRRUS_WORKING_DIR) - - -class TestCirrusYAML(TestCaseBase): - - IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME' - ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES' - - def setUp(self): - TestCirrusYAML._cirrus = None - super().setUp() - - @property - def cirrus(self): - if TestCirrusYAML._cirrus is None: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore",category=DeprecationWarning) - with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml: - TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml) - return TestCirrusYAML._cirrus - - def _assert_get_cache_image_names(self, env): - inames = set([key for key in env.keys() - if key.endswith(self.IMAGE_NAME_SUFFIX)]) - self.assertNotEqual(inames, set()) - - ivalues = set([value for key, value in env.items() - if key in inames]) - self.assertNotEqual(ivalues, set()) - return ivalues - - def _assert_get_subdct(self, key, dct): - self.assertIn(key, dct) - return dct[key] - - def test_parse_yaml(self): - self.assertIsInstance(self.cirrus, dict) - - def test_active_cache_image_names(self): - env = self._assert_get_subdct('env', self.cirrus) - acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env) - - for ivalue in self._assert_get_cache_image_names(env): - self.assertIn(ivalue, acin, - "The '{}' sub-key of 'env' should contain this among" - " its space-separated values." - "".format(self.ACTIVE_IMAGES_NAME)) - - - def test_cache_image_names_active(self): - env = self._assert_get_subdct('env', self.cirrus) - ivalues = self._assert_get_cache_image_names(env) - - for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()): - self.assertIn(avalue, ivalues, - "All space-separated values in the '{}' sub-key" - " of 'env' must also be used in a key with a '{}' suffix." - "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX)) - - -if __name__ == '__main__': - unittest.main(failfast=True, catchbreak=True, verbosity=0) diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 0e8c9e2e2..202663fb7 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -3,9 +3,7 @@ set -e source $(dirname $0)/lib.sh -req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER - -clean_env +req_env_var GOSRC set -x cd "$GOSRC" diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 5e5789cf5..985dbbc74 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -33,9 +33,9 @@ %global shortcommit0 %(c=%{commit0}; echo ${c:0:8}) # People want conmon packaged with the copr rpm -%global import_path_conmon github.com/kubernetes-sigs/cri-o +%global import_path_conmon github.com/containers/conmon %global git_conmon https://%{import_path_conmon} -%global commit_conmon 4cd5a7c60349be0678d9f1b0657683324c1a2726 +%global commit_conmon f02c053eb37010fc76d1e2966de7f2cb9f969ef2 %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman @@ -45,7 +45,7 @@ Summary: Manage Pods, Containers and Container Images License: ASL 2.0 URL: %{git_podman} Source0: %{git0}/archive/%{commit0}/%{repo}-%{shortcommit0}.tar.gz -Source1: crio.tar.gz +Source1: conmon.tar.gz # e.g. el6 has ppc64 arch without gcc-go, so EA tag is required #ExclusiveArch: %%{?go_arches:%%{go_arches}}%%{!?go_arches:%%{ix86} x86_64 aarch64 %%{arm}} ExclusiveArch: aarch64 %{arm} ppc64le s390x x86_64 @@ -371,24 +371,23 @@ GOPATH=$GOPATH go generate ./cmd/podman/varlink/... BUILDTAGS=$BUILDTAGS make binaries docs # build conmon -pushd crio +pushd conmon mkdir _output pushd _output -mkdir -p src/%{provider}.%{provider_tld}/{kubernetes-sigs,opencontainers} +mkdir -p src/%{provider}.%{provider_tld}/{containers,opencontainers} ln -s $(dirs +1 -l) src/%{import_path_conmon} popd -ln -s vendor src -export GOPATH=$(pwd)/_output:$(pwd):%{gopath} export BUILDTAGS="selinux seccomp $(hack/btrfs_installed_tag.sh) $(hack/btrfs_tag.sh) containers_image_ostree_stub" -BUILDTAGS=$BUILDTAGS make -C conmon +BUILDTAGS=$BUILDTAGS make popd %install install -dp %{buildroot}%{_unitdir} PODMAN_VERSION=%{version} %{__make} PREFIX=%{buildroot}%{_prefix} ETCDIR=%{buildroot}%{_sysconfdir} \ install.bin \ + install.remote \ install.man \ install.cni \ install.systemd \ @@ -402,7 +401,7 @@ install -p -m 644 %{repo}.conf %{buildroot}%{_datadir}/containers # install conmon install -dp %{buildroot}%{_libexecdir}/%{name} -install -p -m 755 crio/bin/conmon %{buildroot}%{_libexecdir}/%{name} +install -p -m 755 conmon/bin/conmon %{buildroot}%{_libexecdir}/%{name} # source codes for building projects %if 0%{?with_devel} diff --git a/docs/libpod.conf.5.md b/docs/libpod.conf.5.md index 2f0b3f303..cb08f0eb0 100644 --- a/docs/libpod.conf.5.md +++ b/docs/libpod.conf.5.md @@ -34,7 +34,7 @@ libpod to manage containers. Each `*.json` file in the path configures a hook for Podman containers. For more details on the syntax of the JSON files and the semantics of hook injection, see `oci-hooks(5)`. Podman and libpod currently support both the 1.0.0 and 0.1.0 hook schemas, although the 0.1.0 schema is deprecated. - Paths listed later in the array higher precedence (`oci-hooks(5)` discusses directory precedence). + Paths listed later in the array have higher precedence (`oci-hooks(5)` discusses directory precedence). For the annotation conditions, libpod uses any annotations set in the generated OCI configuration. diff --git a/docs/podman-build.1.md b/docs/podman-build.1.md index ccc8bd900..1c96d3d5f 100644 --- a/docs/podman-build.1.md +++ b/docs/podman-build.1.md @@ -506,6 +506,8 @@ You can add the `:ro` or `:rw` suffix to a volume to mount it read-only or read-write mode, respectively. By default, the volumes are mounted read-write. See examples. + `Labeling Volume Mounts` + Labeling systems like SELinux require that proper labels are placed on volume content mounted into a container. Without a label, the security system might prevent the processes running inside the container from using the content. By @@ -519,6 +521,21 @@ content label. Shared volume labels allow all containers to read/write content. The `Z` option tells podman to label the content with a private unshared label. Only the current container can use a private volume. + `Overlay Volume Mounts` + + The `:O` flag tells Buildah to mount the directory from the host as a temporary storage using the Overlay file system. The `RUN` command containers are allowed to modify contents within the mountpoint and are stored in the container storage in a separate directory. In Ovelay FS terms the source directory will be the lower, and the container storage directory will be the upper. Modifications to the mount point are destroyed when the `RUN` command finishes executing, similar to a tmpfs mount point. + + Any subsequent execution of `RUN` commands sees the original source directory content, any changes from previous RUN commands no longer exists. + + One use case of the `overlay` mount is sharing the package cache from the host into the container to allow speeding up builds. + + Note: + + - Overlay mounts are not currently supported in rootless mode. + - The `O` flag is not allowed to be specified with the `Z` or `z` flags. Content mounted into the container is labeled with the private label. + On SELinux systems, labels in the source directory needs to be readable by the container label. If not, SELinux container separation must be disabled for the container to work. + - Modification of the directory volume mounted into the container with an overlay mount can cause unexpected failures. It is recommended that you do not modify the directory until the container finishes running. + By default bind mounted volumes are `private`. That means any mounts done inside container will not be visible on the host and vice versa. This behavior can be changed by specifying a volume mount propagation property. @@ -583,6 +600,8 @@ $ podman build --security-opt label=level:s0:c100,c200 --cgroup-parent /path/to/ $ podman build --volume /home/test:/myvol:ro,Z -t imageName . +$ podman build -v /var/lib/yum:/var/lib/yum:O -t imageName . + $ podman build --layers -t imageName . $ podman build --no-cache -t imageName . diff --git a/docs/podman-run.1.md b/docs/podman-run.1.md index 1840e0f0b..210ed4f8a 100644 --- a/docs/podman-run.1.md +++ b/docs/podman-run.1.md @@ -291,16 +291,16 @@ The initialization time needed for a container to bootstrap. The value can be ex The maximum time allowed to complete the healthcheck before an interval is considered failed. Like start-period, the value can be expressed in a time format such as `1m22s`. The default value is `30s`. +**--help** + +Print usage statement + **--hostname**="" Container host name Sets the container host name that is available inside the container. -**--help** - -Print usage statement - **--http-proxy**=*true*|*false* By default proxy environment variables are passed into the container if set @@ -769,25 +769,11 @@ This option is incompatible with --gidmap, --uidmap, --subuid and --subgid Set the UTS mode for the container -`host`: use the host's UTS namespace inside the container. -`ns`: specify the user namespace to use. +- `host`: use the host's UTS namespace inside the container. +- `ns`: specify the user namespace to use. **NOTE**: the host mode gives the container access to changing the host's hostname and is therefore considered insecure. -**--userns**="" - -Set the user namespace mode for the container. The use of userns is disabled by default. - - **host**: use the host user namespace and enable all privileged options (e.g., `pid=host` or `--privileged`). - **ns**: specify the user namespace to use. - -**--uts**=*host* - -Set the UTS mode for the container - **host**: use the host's UTS namespace inside the container. - **ns**: specify the user namespace to use. - Note: the host mode gives the container access to changing the host's hostname and is therefore considered insecure. - **--volume**, **-v**[=*[HOST-DIR:CONTAINER-DIR[:OPTIONS]]*] Create a bind mount. If you specify, ` -v /HOST-DIR:/CONTAINER-DIR`, podman diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index d0325b8ae..370cd8a5e 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -19,7 +19,6 @@ PROJECT="libpod-218412" GOSRC="/var/tmp/go/src/github.com/containers/libpod" GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest} GCLOUD_SUDO=${GCLOUD_SUDO-sudo} -ROOTLESS_USER="madcowdog" # Shared tmp directory between container and us TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX) @@ -48,11 +47,15 @@ showrun() { } cleanup() { + RET=$? set +e wait # set GCLOUD_DEBUG to leave tmpdir behind for postmortem test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR + + # Not always called from an exit handler, but should always exit when called + exit $RET } trap cleanup EXIT @@ -67,14 +70,15 @@ delvm() { image_hints() { egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \ "$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \ - grep -v 'notready' | grep -v 'image-builder' | sort -u + grep -v 'notready' | sort -u } show_usage() { echo -e "\n${RED}ERROR: $1${NOR}" - echo -e "${YEL}Usage: $(basename $0) [-s | -p | -r] <image_name>${NOR}" - echo "Use -s / -p to select source or package based dependencies" - echo -e "Use -r to setup and run tests as a regular user.\n" + echo -e "${YEL}Usage: $(basename $0) [-m <SPECIALMODE>] [-u <ROOTLESS_USER> ] <image_name>${NOR}" + echo "Use -m <SPECIALMODE> with a supported value documented in contrib/cirrus/README.md." + echo "With '-m rootless' must also specify -u <ROOTLESS_USER> with name of user to create & use" + echo "" if [[ -r ".cirrus.yml" ]] then echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}" @@ -87,7 +91,7 @@ show_usage() { get_env_vars() { python -c ' import yaml -env=yaml.load(open(".cirrus.yml"))["env"] +env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"] keys=[k for k in env if "ENCRYPTED" not in str(env[k])] for k,v in env.items(): v=str(v) @@ -99,28 +103,56 @@ for k,v in env.items(): parse_args(){ echo -e "$USAGE_WARNING" - if [[ -z "$1" ]] + if [[ "$USER" =~ "root" ]] then + show_usage "This script must be run as a regular user." + fi + + ENVS="$(get_env_vars)" + [[ "$#" -ge "1" ]] || \ show_usage "Must specify at least one command-line parameter." - elif [[ "$1" == "-p" ]] - then - echo -e "${YEL}Hint: Use -p for package-based dependencies or -s for source-based.${NOR}" - DEPS="PACKAGE_DEPS=true SOURCE_DEPS=false" - IMAGE_NAME="$2" - elif [[ "$1" == "-s" ]] + IMAGE_NAME="" + ROOTLESS_USER="" + SPECIALMODE="none" + for arg + do + if [[ "$SPECIALMODE" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]] + then + SPECIALMODE="$arg" + echo -e "${YEL}Using \$SPECIALMODE=$SPECIALMODE.${NOR}" + continue + elif [[ "$ROOTLESS_USER" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]] + then + ROOTLESS_USER="$arg" + echo -e "${YEL}Using \$ROOTLESS_USER=$ROOTLESS_USER.${NOR}" + continue + fi + case "$arg" in + -m) + SPECIALMODE="GRABNEXT" + ;; + -u) + ROOTLESS_USER="GRABNEXT" + ;; + *) + [[ "${arg:0:1}" != "-" ]] || \ + show_usage "Unknown command-line option '$arg'." + [[ -z "$IMAGE_NAME" ]] || \ + show_usage "Must specify exactly one image name, got '$IMAGE_NAME' and '$arg'." + IMAGE_NAME="$arg" + ;; + esac + done + + if [[ "$SPECIALMODE" == "GRABNEXT" ]] then - echo -e "${RED}Using source-based dependencies.${NOR}" - DEPS="PACKAGE_DEPS=false SOURCE_DEPS=true" - IMAGE_NAME="$2" - elif [[ "$1" == "-r" ]] + show_usage "Must specify argument to -m option." + fi + + if [[ "$ROOTLESS_USER" == "GRABNEXT" ]] then - DEPS="ROOTLESS_USER=$ROOTLESS_USER" - IMAGE_NAME="$2" - else # no -s or -p - echo -e "${RED}Using package-based dependencies.${NOR}" - DEPS="$(get_env_vars)" - IMAGE_NAME="$1" + show_usage "Must specify argument to -u option." fi if [[ -z "$IMAGE_NAME" ]] @@ -128,15 +160,33 @@ parse_args(){ show_usage "No image-name specified." fi - if [[ "$USER" =~ "root" ]] + if [[ "$SPECIALMODE" == "rootless" ]] && [[ -z "$ROOTLESS_USER" ]] then - show_usage "This script must be run as a regular user." + show_usage "With '-m rootless' must also pass -u <username> of rootless user." + fi + + if echo "$IMAGE_NAME" | grep -q "image-builder-image" + then + echo -e "Creating an image-builder VM, I hope you know what you're doing.\n" + IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\"" + SSHUSER="centos" + else + unset IBI_ARGS + SSHUSER="root" fi - SETUP_CMD="env $DEPS $GOSRC/contrib/cirrus/setup_environment.sh" + ENVS="$ENVS SPECIALMODE=\"$SPECIALMODE\"" + + [[ -z "$ROOTLESS_USER" ]] || \ + ENVS="$ENVS ROOTLESS_USER=$ROOTLESS_USER" + + SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup_environment.sh" VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}" - CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME" - SSH_CMD="$PGCLOUD compute ssh root@$VMNAME" + + CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $IBI_ARGS $VMNAME" + + SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME" + CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME" } @@ -147,7 +197,7 @@ parse_args(){ cd "$LIBPODROOT" -parse_args $@ +parse_args "$@" # Ensure mount-points and data directories exist on host as $USER. Also prevents # permission-denied errors during cleanup() b/c `sudo podman` created mount-points @@ -188,7 +238,7 @@ showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed -C $LIBP trap delvm INT # Allow deleting VM if CTRL-C during create # This fails if VM already exists: permit this usage to re-init -echo -e "\n${YEL}Trying to creating a VM named $VMNAME ${RED}(might take a minute/two. Errors ignored).${NOR}" +echo -e "\n${YEL}Trying to creating a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}" showrun $CREATE_CMD || true # allow re-running commands below when "delete: N" # Any subsequent failure should prompt for VM deletion @@ -208,19 +258,13 @@ then fi echo -e "${YEL}Got it${NOR}" -if $SSH_CMD --command "test -r /root/.bash_profile_original" -then - echo -e "\n${YEL}Resetting environment configuration${NOR}" - showrun $SSH_CMD --command "cp /root/.bash_profile_original /root/.bash_profile" -fi - echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}" showrun $SSH_CMD --command "rm -rf $GOSRC" showrun $SSH_CMD --command "mkdir -p $GOSRC" echo -e "\n${YEL}Transfering tarball to $VMNAME.${NOR}" wait -showrun $SCP_CMD $HOME/$TARBALL root@$VMNAME:/tmp/$TARBALL +showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}" showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC" @@ -231,9 +275,12 @@ showrun $SSH_CMD --command "rm -f /tmp/$TARBALL" echo -e "\n${YEL}Executing environment setup${NOR}" showrun $SSH_CMD --command "$SETUP_CMD" -echo -e "\n${YEL}Connecting to $VMNAME ${RED}(option to delete VM upon logout).${NOR}\n" -if [[ "$1" == "-r" ]] +echo -e "\n${YEL}Connecting to $VMNAME\n${RED}(option to delete VM upon logout).${NOR}\n" +if [[ -n "$ROOTLESS_USER" ]] then + echo "Re-chowning source files after transfer" + showrun $SSH_CMD --command "chown -R $ROOTLESS_USER $GOSRC" + echo "Connecting as user $ROOTLESS_USER" SSH_CMD="$PGCLOUD compute ssh $ROOTLESS_USER@$VMNAME" fi -showrun $SSH_CMD -- -t "cd $GOSRC && exec env $DEPS bash -il" +showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il" diff --git a/install.md b/install.md index 82dd4c36a..ae74acdf8 100644 --- a/install.md +++ b/install.md @@ -91,7 +91,6 @@ Fedora, CentOS, RHEL, and related distributions: sudo yum install -y \ atomic-registries \ btrfs-progs-devel \ - conmon \ containernetworking-cni \ device-mapper-devel \ git \ @@ -188,13 +187,12 @@ export PATH=$GOPATH/bin:$PATH #### conmon The latest version of `conmon` is expected to be installed on the system. Conmon is used to monitor OCI Runtimes. -To build from source, use the following (if not already executed above, run `export GOPATH=~/go && mkdir -p $GOPATH`): +To build from source, use the following: ```bash -git clone https://github.com/cri-o/cri-o $GOPATH/src/github.com/cri-o/cri-o -cd $GOPATH/src/github.com/cri-o/cri-o -mkdir bin -make bin/conmon +git clone https://github.com/containers/conmon +cd conmon +make sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon ``` diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index d8cfa2bda..63e40a98f 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -5,7 +5,7 @@ import ( "strings" "sync" - "github.com/boltdb/bolt" + bolt "github.com/etcd-io/bbolt" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go index a6900a6d3..313e5f4d7 100644 --- a/libpod/boltdb_state_internal.go +++ b/libpod/boltdb_state_internal.go @@ -5,9 +5,9 @@ import ( "runtime" "strings" - "github.com/boltdb/bolt" "github.com/containers/libpod/pkg/rootless" "github.com/containers/storage" + bolt "github.com/etcd-io/bbolt" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/libpod/container_api.go b/libpod/container_api.go index 06a31da11..eff5bfe5f 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -289,8 +289,8 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir chWait := make(chan error) go func() { chWait <- execCmd.Wait() + close(chWait) }() - defer close(chWait) pidFile := c.execPidPath(sessionID) // 60 second seems a reasonable time to wait diff --git a/libpod/container_commit.go b/libpod/container_commit.go index ae04f67bb..739fcd80e 100644 --- a/libpod/container_commit.go +++ b/libpod/container_commit.go @@ -99,7 +99,7 @@ func (c *Container) Commit(ctx context.Context, destImage string, options Contai // Should we store the ENV we actually want in the spec separately? if c.config.Spec.Process != nil { for _, e := range c.config.Spec.Process.Env { - splitEnv := strings.Split(e, "=") + splitEnv := strings.SplitN(e, "=", 2) importBuilder.SetEnv(splitEnv[0], splitEnv[1]) } } diff --git a/libpod/util.go b/libpod/util.go index 7e2dff21a..3a15f9e39 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -90,11 +90,7 @@ func MountExists(specMounts []spec.Mount, dest string) bool { // WaitForFile waits until a file has been created or the given timeout has occurred func WaitForFile(path string, chWait chan error, timeout time.Duration) (bool, error) { - done := make(chan struct{}) - chControl := make(chan struct{}) - var inotifyEvents chan fsnotify.Event - var timer chan struct{} watcher, err := fsnotify.NewWatcher() if err == nil { if err := watcher.Add(filepath.Dir(path)); err == nil { @@ -102,51 +98,36 @@ func WaitForFile(path string, chWait chan error, timeout time.Duration) (bool, e } defer watcher.Close() } - if inotifyEvents == nil { - // If for any reason we fail to create the inotify - // watcher, fallback to polling the file - timer = make(chan struct{}) - go func() { - select { - case <-chControl: - close(timer) - return - default: - time.Sleep(25 * time.Millisecond) - timer <- struct{}{} - } - }() - } - go func() { - for { - select { - case <-chControl: - return - case <-timer: - _, err := os.Stat(path) - if err == nil { - close(done) - return - } - case <-inotifyEvents: - _, err := os.Stat(path) - if err == nil { - close(done) - return - } + timeoutChan := time.After(timeout) + + for { + select { + case e := <-chWait: + return true, e + case <-inotifyEvents: + _, err := os.Stat(path) + if err == nil { + return false, nil + } + if !os.IsNotExist(err) { + return false, errors.Wrapf(err, "checking file %s", path) + } + case <-time.After(25 * time.Millisecond): + // Check periodically for the file existence. It is needed + // if the inotify watcher could not have been created. It is + // also useful when using inotify as if for any reasons we missed + // a notification, we won't hang the process. + _, err := os.Stat(path) + if err == nil { + return false, nil + } + if !os.IsNotExist(err) { + return false, errors.Wrapf(err, "checking file %s", path) } + case <-timeoutChan: + return false, errors.Wrapf(ErrInternal, "timed out waiting for file %s", path) } - }() - - select { - case e := <-chWait: - return true, e - case <-done: - return false, nil - case <-time.After(timeout): - close(chControl) - return false, errors.Wrapf(ErrInternal, "timed out waiting for file %s", path) } } diff --git a/pkg/adapter/client.go b/pkg/adapter/client.go index f672a92a6..01914834f 100644 --- a/pkg/adapter/client.go +++ b/pkg/adapter/client.go @@ -10,44 +10,56 @@ import ( "github.com/varlink/go/varlink" ) -type VarlinkConnectionInfo struct { - RemoteUserName string - RemoteHost string - VarlinkAddress string -} - -// Connect provides a varlink connection -func (r RemoteRuntime) Connect() (*varlink.Connection, error) { - var ( - err error - connection *varlink.Connection - ) +var remoteEndpoint *Endpoint - logLevel := r.cmd.LogLevel +func (r RemoteRuntime) RemoteEndpoint() (remoteEndpoint *Endpoint, err error) { + if remoteEndpoint == nil { + remoteEndpoint = &Endpoint{Unknown, ""} + } else { + return remoteEndpoint, nil + } // I'm leaving this here for now as a document of the birdge format. It can be removed later once the bridge // function is more flushed out. - //bridge := `ssh -T root@192.168.122.1 "/usr/bin/varlink -A '/usr/bin/podman varlink \$VARLINK_ADDRESS' bridge"` + // bridge := `ssh -T root@192.168.122.1 "/usr/bin/varlink -A '/usr/bin/podman varlink \$VARLINK_ADDRESS' bridge"` if len(r.cmd.RemoteHost) > 0 { // The user has provided a remote host endpoint if len(r.cmd.RemoteUserName) < 1 { return nil, errors.New("you must provide a username when providing a remote host name") } - bridge := fmt.Sprintf(`ssh -T %s@%s /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, r.cmd.RemoteUserName, r.cmd.RemoteHost, logLevel) - connection, err = varlink.NewBridge(bridge) + remoteEndpoint.Type = BridgeConnection + remoteEndpoint.Connection = fmt.Sprintf( + `ssh -T %s@%s /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, + r.cmd.RemoteUserName, r.cmd.RemoteHost, r.cmd.LogLevel) + } else if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" { - connection, err = varlink.NewBridge(bridge) + remoteEndpoint.Type = BridgeConnection + remoteEndpoint.Connection = bridge } else { address := os.Getenv("PODMAN_VARLINK_ADDRESS") if address == "" { address = DefaultAddress } - connection, err = varlink.NewConnection(address) + remoteEndpoint.Type = DirectConnection + remoteEndpoint.Connection = address } + return +} + +// Connect provides a varlink connection +func (r RemoteRuntime) Connect() (*varlink.Connection, error) { + ep, err := r.RemoteEndpoint() if err != nil { return nil, err } - return connection, nil + + switch ep.Type { + case DirectConnection: + return varlink.NewConnection(ep.Connection) + case BridgeConnection: + return varlink.NewBridge(ep.Connection) + } + return nil, errors.New(fmt.Sprintf("Unable to determine type of varlink connection: %s", ep.Connection)) } // RefreshConnection is used to replace the current r.Conn after things like diff --git a/pkg/adapter/client_config.go b/pkg/adapter/client_config.go index d165ef1cc..3559b16e3 100644 --- a/pkg/adapter/client_config.go +++ b/pkg/adapter/client_config.go @@ -2,3 +2,35 @@ package adapter // DefaultAddress is the default address of the varlink socket const DefaultAddress = "unix:/run/podman/io.podman" + +// EndpointType declares the type of server connection +type EndpointType int + +// Enum of connection types +const ( + Unknown = iota - 1 // Unknown connection type + BridgeConnection // BridgeConnection proxy connection via ssh + DirectConnection // DirectConnection socket connection to server +) + +// String prints ASCII string for EndpointType +func (e EndpointType) String() string { + // declare an array of strings + // ... operator counts how many + // items in the array (7) + names := [...]string{ + "BridgeConnection", + "DirectConnection", + } + + if e < BridgeConnection || e > DirectConnection { + return "Unknown" + } + return names[e] +} + +// Endpoint type and connection string to use +type Endpoint struct { + Type EndpointType + Connection string +} diff --git a/pkg/adapter/runtime.go b/pkg/adapter/runtime.go index 21613c425..37ee1b737 100644 --- a/pkg/adapter/runtime.go +++ b/pkg/adapter/runtime.go @@ -398,3 +398,8 @@ func (r *LocalRuntime) GetPodsByStatus(statuses []string) ([]*libpod.Pod, error) func (r *LocalRuntime) GetVersion() (libpod.Version, error) { return libpod.GetVersion() } + +// RemoteEndpoint resolve interface requirement +func (r *LocalRuntime) RemoteEndpoint() (*Endpoint, error) { + return nil, errors.New("RemoteEndpoint() not implemented for local connection") +} diff --git a/pkg/hooks/0.1.0/hook.go b/pkg/hooks/0.1.0/hook.go index 49d833aa8..ba68b0f10 100644 --- a/pkg/hooks/0.1.0/hook.go +++ b/pkg/hooks/0.1.0/hook.go @@ -19,7 +19,7 @@ type Hook struct { Hook *string `json:"hook"` Arguments []string `json:"arguments,omitempty"` - // https://github.com/kubernetes-sigs/cri-o/pull/1235 + // https://github.com/cri-o/cri-o/pull/1235 Stages []string `json:"stages"` Stage []string `json:"stage"` diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go index fdb7f33c5..5c4ecd020 100644 --- a/pkg/registries/registries.go +++ b/pkg/registries/registries.go @@ -51,7 +51,7 @@ func GetRegistries() ([]string, error) { } for _, reg := range registries { if reg.Search { - searchRegistries = append(searchRegistries, reg.URL) + searchRegistries = append(searchRegistries, reg.Location) } } return searchRegistries, nil @@ -66,7 +66,7 @@ func GetBlockedRegistries() ([]string, error) { } for _, reg := range registries { if reg.Blocked { - blockedRegistries = append(blockedRegistries, reg.URL) + blockedRegistries = append(blockedRegistries, reg.Location) } } return blockedRegistries, nil @@ -81,7 +81,7 @@ func GetInsecureRegistries() ([]string, error) { } for _, reg := range registries { if reg.Insecure { - insecureRegistries = append(insecureRegistries, reg.URL) + insecureRegistries = append(insecureRegistries, reg.Location) } } return insecureRegistries, nil diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go index c2c5e0900..0d953ff6f 100644 --- a/pkg/spec/spec.go +++ b/pkg/spec/spec.go @@ -268,7 +268,9 @@ func (config *CreateConfig) createConfigToOCISpec(runtime *libpod.Runtime, userM // SECURITY OPTS g.SetProcessNoNewPrivileges(config.NoNewPrivs) - g.SetProcessApparmorProfile(config.ApparmorProfile) + if !config.Privileged { + g.SetProcessApparmorProfile(config.ApparmorProfile) + } blockAccessToKernelFilesystems(config, &g) @@ -355,6 +357,10 @@ func (config *CreateConfig) createConfigToOCISpec(runtime *libpod.Runtime, userM if addedResources && !cgroup2 { return nil, errors.New("invalid configuration, cannot set resources with rootless containers not using cgroups v2 unified mode") } + if !cgroup2 { + // Force the resources block to be empty instead of having default values. + configSpec.Linux.Resources = &spec.LinuxResources{} + } } // Make sure that the bind mounts keep options like nosuid, noexec, nodev. diff --git a/test/e2e/commit_test.go b/test/e2e/commit_test.go index 3ece4887e..bf20ac999 100644 --- a/test/e2e/commit_test.go +++ b/test/e2e/commit_test.go @@ -194,4 +194,24 @@ var _ = Describe("Podman commit", func() { Expect(r.ExitCode()).To(Equal(0)) }) + It("podman commit container check env variables", func() { + s := podmanTest.Podman([]string{"run", "--name", "test1", "-e", "TEST=1=1-01=9.01", "-it", "alpine", "true"}) + s.WaitWithDefaultTimeout() + Expect(s.ExitCode()).To(Equal(0)) + + c := podmanTest.Podman([]string{"commit", "test1", "newimage"}) + c.WaitWithDefaultTimeout() + Expect(c.ExitCode()).To(Equal(0)) + + inspect := podmanTest.Podman([]string{"inspect", "newimage"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect.ExitCode()).To(Equal(0)) + image := inspect.InspectImageJSON() + + envMap := make(map[string]bool) + for _, v := range image[0].Config.Env { + envMap[v] = true + } + Expect(envMap["TEST=1=1-01=9.01"]).To(BeTrue()) + }) }) diff --git a/test/e2e/images_test.go b/test/e2e/images_test.go index bec6e304b..23455163b 100644 --- a/test/e2e/images_test.go +++ b/test/e2e/images_test.go @@ -298,4 +298,17 @@ ENV foo=bar Expect(session2.ExitCode()).To(Equal(0)) Expect(len(session2.OutputToStringArray())).To(Equal(6)) }) + + It("podman images filter by label", func() { + SkipIfRemote() + dockerfile := `FROM docker.io/library/alpine:latest +LABEL version="1.0" +LABEL "com.example.vendor"="Example Vendor" +` + podmanTest.BuildImage(dockerfile, "test", "true") + session := podmanTest.Podman([]string{"images", "-f", "label=version=1.0"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + }) }) diff --git a/test/registries.conf b/test/registries.conf index 6c9d39bbc..bb7072d45 100644 --- a/test/registries.conf +++ b/test/registries.conf @@ -1,5 +1,5 @@ [registries.search] -registries = ['docker.io', 'quay.io'] +registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org'] [registries.insecure] registries = [] diff --git a/transfer.md b/transfer.md index df91cdf21..79b6d3461 100644 --- a/transfer.md +++ b/transfer.md @@ -98,7 +98,7 @@ Those Docker commands currently do not have equivalents in `podman`: | `docker secret` || | `docker service` || | `docker stack` || -| `docker swarm` | podman does not support swarm. We support Kubernetes for orchestration using [CRI-O](https://github.com/kubernetes-sigs/cri-o).| +| `docker swarm` | podman does not support swarm. We support Kubernetes for orchestration using [CRI-O](https://github.com/cri-o/cri-o).| | `docker volume` | podman currently supports file volumes. Future enhancement planned to support Docker Volumes Plugins ## Missing commands in Docker diff --git a/vendor.conf b/vendor.conf index 0b1f13304..f9b7b128d 100644 --- a/vendor.conf +++ b/vendor.conf @@ -7,7 +7,7 @@ github.com/BurntSushi/toml v0.2.0 github.com/Microsoft/go-winio v0.4.11 github.com/Microsoft/hcsshim v0.8.3 github.com/blang/semver v3.5.0 -github.com/boltdb/bolt v1.3.1 +github.com/etcd-io/bbolt v1.3.2 # TODO: no release, can we find an alternative? github.com/buger/goterm c206103e1f37c0c6c5c039706305ea2aa6e8ad3b github.com/checkpoint-restore/go-criu v3.11 @@ -15,7 +15,7 @@ github.com/containerd/cgroups 4994991857f9b0ae8dc439551e8bebdbb4bf66c1 github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containernetworking/cni v0.7.0-rc2 github.com/containernetworking/plugins v0.7.4 -github.com/containers/image v1.5.1 +github.com/containers/image 2c0349c99af7d90694b3faa0e9bde404d407b145 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 @@ -93,7 +93,7 @@ k8s.io/api kubernetes-1.10.13-beta.0 https://github.com/kubernetes/api k8s.io/apimachinery kubernetes-1.10.13-beta.0 https://github.com/kubernetes/apimachinery k8s.io/client-go kubernetes-1.10.13-beta.0 https://github.com/kubernetes/client-go github.com/mrunalp/fileutils 7d4729fb36185a7c1719923406c9d40e54fb93c7 -github.com/containers/buildah v1.8.2 +github.com/containers/buildah bcc5e51a948c1847fab1c932f1a343696a96cafd github.com/varlink/go 0f1d566d194b9d6d48e0d47c5e4d822628919066 # TODO: Gotty has not been updated since 2012. Can we find replacement? github.com/Nvveen/Gotty cd527374f1e5bff4938207604a14f2e38a9cf512 diff --git a/vendor/github.com/boltdb/bolt/freelist.go b/vendor/github.com/boltdb/bolt/freelist.go deleted file mode 100644 index aba48f58c..000000000 --- a/vendor/github.com/boltdb/bolt/freelist.go +++ /dev/null @@ -1,252 +0,0 @@ -package bolt - -import ( - "fmt" - "sort" - "unsafe" -) - -// freelist represents a list of all pages that are available for allocation. -// It also tracks pages that have been freed but are still in use by open transactions. -type freelist struct { - ids []pgid // all free and available free page ids. - pending map[txid][]pgid // mapping of soon-to-be free page ids by tx. - cache map[pgid]bool // fast lookup of all free and pending page ids. -} - -// newFreelist returns an empty, initialized freelist. -func newFreelist() *freelist { - return &freelist{ - pending: make(map[txid][]pgid), - cache: make(map[pgid]bool), - } -} - -// size returns the size of the page after serialization. -func (f *freelist) size() int { - n := f.count() - if n >= 0xFFFF { - // The first element will be used to store the count. See freelist.write. - n++ - } - return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n) -} - -// count returns count of pages on the freelist -func (f *freelist) count() int { - return f.free_count() + f.pending_count() -} - -// free_count returns count of free pages -func (f *freelist) free_count() int { - return len(f.ids) -} - -// pending_count returns count of pending pages -func (f *freelist) pending_count() int { - var count int - for _, list := range f.pending { - count += len(list) - } - return count -} - -// copyall copies into dst a list of all free ids and all pending ids in one sorted list. -// f.count returns the minimum length required for dst. -func (f *freelist) copyall(dst []pgid) { - m := make(pgids, 0, f.pending_count()) - for _, list := range f.pending { - m = append(m, list...) - } - sort.Sort(m) - mergepgids(dst, f.ids, m) -} - -// allocate returns the starting page id of a contiguous list of pages of a given size. -// If a contiguous block cannot be found then 0 is returned. -func (f *freelist) allocate(n int) pgid { - if len(f.ids) == 0 { - return 0 - } - - var initial, previd pgid - for i, id := range f.ids { - if id <= 1 { - panic(fmt.Sprintf("invalid page allocation: %d", id)) - } - - // Reset initial page if this is not contiguous. - if previd == 0 || id-previd != 1 { - initial = id - } - - // If we found a contiguous block then remove it and return it. - if (id-initial)+1 == pgid(n) { - // If we're allocating off the beginning then take the fast path - // and just adjust the existing slice. This will use extra memory - // temporarily but the append() in free() will realloc the slice - // as is necessary. - if (i + 1) == n { - f.ids = f.ids[i+1:] - } else { - copy(f.ids[i-n+1:], f.ids[i+1:]) - f.ids = f.ids[:len(f.ids)-n] - } - - // Remove from the free cache. - for i := pgid(0); i < pgid(n); i++ { - delete(f.cache, initial+i) - } - - return initial - } - - previd = id - } - return 0 -} - -// free releases a page and its overflow for a given transaction id. -// If the page is already free then a panic will occur. -func (f *freelist) free(txid txid, p *page) { - if p.id <= 1 { - panic(fmt.Sprintf("cannot free page 0 or 1: %d", p.id)) - } - - // Free page and all its overflow pages. - var ids = f.pending[txid] - for id := p.id; id <= p.id+pgid(p.overflow); id++ { - // Verify that page is not already free. - if f.cache[id] { - panic(fmt.Sprintf("page %d already freed", id)) - } - - // Add to the freelist and cache. - ids = append(ids, id) - f.cache[id] = true - } - f.pending[txid] = ids -} - -// release moves all page ids for a transaction id (or older) to the freelist. -func (f *freelist) release(txid txid) { - m := make(pgids, 0) - for tid, ids := range f.pending { - if tid <= txid { - // Move transaction's pending pages to the available freelist. - // Don't remove from the cache since the page is still free. - m = append(m, ids...) - delete(f.pending, tid) - } - } - sort.Sort(m) - f.ids = pgids(f.ids).merge(m) -} - -// rollback removes the pages from a given pending tx. -func (f *freelist) rollback(txid txid) { - // Remove page ids from cache. - for _, id := range f.pending[txid] { - delete(f.cache, id) - } - - // Remove pages from pending list. - delete(f.pending, txid) -} - -// freed returns whether a given page is in the free list. -func (f *freelist) freed(pgid pgid) bool { - return f.cache[pgid] -} - -// read initializes the freelist from a freelist page. -func (f *freelist) read(p *page) { - // If the page.count is at the max uint16 value (64k) then it's considered - // an overflow and the size of the freelist is stored as the first element. - idx, count := 0, int(p.count) - if count == 0xFFFF { - idx = 1 - count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0]) - } - - // Copy the list of page ids from the freelist. - if count == 0 { - f.ids = nil - } else { - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) - - // Make sure they're sorted. - sort.Sort(pgids(f.ids)) - } - - // Rebuild the page cache. - f.reindex() -} - -// write writes the page ids onto a freelist page. All free and pending ids are -// saved to disk since in the event of a program crash, all pending ids will -// become free. -func (f *freelist) write(p *page) error { - // Combine the old free pgids and pgids waiting on an open transaction. - - // Update the header flag. - p.flags |= freelistPageFlag - - // The page.count can only hold up to 64k elements so if we overflow that - // number then we handle it by putting the size in the first element. - lenids := f.count() - if lenids == 0 { - p.count = uint16(lenids) - } else if lenids < 0xFFFF { - p.count = uint16(lenids) - f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:]) - } else { - p.count = 0xFFFF - ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids) - f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:]) - } - - return nil -} - -// reload reads the freelist from a page and filters out pending items. -func (f *freelist) reload(p *page) { - f.read(p) - - // Build a cache of only pending pages. - pcache := make(map[pgid]bool) - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { - pcache[pendingID] = true - } - } - - // Check each page in the freelist and build a new available freelist - // with any pages not in the pending lists. - var a []pgid - for _, id := range f.ids { - if !pcache[id] { - a = append(a, id) - } - } - f.ids = a - - // Once the available list is rebuilt then rebuild the free cache so that - // it includes the available and pending free pages. - f.reindex() -} - -// reindex rebuilds the free cache based on available and pending free lists. -func (f *freelist) reindex() { - f.cache = make(map[pgid]bool, len(f.ids)) - for _, id := range f.ids { - f.cache[id] = true - } - for _, pendingIDs := range f.pending { - for _, pendingID := range pendingIDs { - f.cache[pendingID] = true - } - } -} diff --git a/vendor/github.com/containers/buildah/add.go b/vendor/github.com/containers/buildah/add.go index d67a481f1..589e090a8 100644 --- a/vendor/github.com/containers/buildah/add.go +++ b/vendor/github.com/containers/buildah/add.go @@ -15,6 +15,7 @@ import ( "github.com/containers/buildah/util" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -88,7 +89,7 @@ func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer) // filesystem, optionally extracting contents of local files that look like // non-empty archives. func (b *Builder) Add(destination string, extract bool, options AddAndCopyOptions, source ...string) error { - excludes := DockerIgnoreHelper(options.Excludes, options.ContextDir) + excludes := dockerIgnoreHelper(options.Excludes, options.ContextDir) mountPoint, err := b.Mount(b.MountLabel) if err != nil { return err @@ -177,16 +178,16 @@ func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { return u, err } -// DockerIgnore struct keep info from .dockerignore -type DockerIgnore struct { +// dockerIgnore struct keep info from .dockerignore +type dockerIgnore struct { ExcludePath string IsExcluded bool } -// DockerIgnoreHelper returns the lines from .dockerignore file without the comments +// dockerIgnoreHelper returns the lines from .dockerignore file without the comments // and reverses the order -func DockerIgnoreHelper(lines []string, contextDir string) []DockerIgnore { - var excludes []DockerIgnore +func dockerIgnoreHelper(lines []string, contextDir string) []dockerIgnore { + var excludes []dockerIgnore // the last match of a file in the .dockerignmatches determines whether it is included or excluded // reverse the order for i := len(lines) - 1; i >= 0; i-- { @@ -200,15 +201,15 @@ func DockerIgnoreHelper(lines []string, contextDir string) []DockerIgnore { exclude = strings.TrimPrefix(exclude, "!") excludeFlag = false } - excludes = append(excludes, DockerIgnore{ExcludePath: filepath.Join(contextDir, exclude), IsExcluded: excludeFlag}) + excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, exclude), IsExcluded: excludeFlag}) } if len(excludes) != 0 { - excludes = append(excludes, DockerIgnore{ExcludePath: filepath.Join(contextDir, ".dockerignore"), IsExcluded: true}) + excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, ".dockerignore"), IsExcluded: true}) } return excludes } -func addHelper(excludes []DockerIgnore, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { +func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { dirsInDockerignore, err := getDirsInDockerignore(options.ContextDir, excludes) if err != nil { return errors.Wrapf(err, "error checking directories in .dockerignore") @@ -270,9 +271,6 @@ func addHelper(excludes []DockerIgnore, extract bool, dest string, destfi os.Fil if err != nil { return err } - if info.IsDir() { - return nil - } for _, exclude := range excludes { match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) if err != nil { @@ -292,7 +290,25 @@ func addHelper(excludes []DockerIgnore, extract bool, dest string, destfi os.Fil break } // combine the filename with the dest directory - fpath := strings.TrimPrefix(path, esrc) + fpath, err := filepath.Rel(esrc, path) + if err != nil { + return errors.Wrapf(err, "error converting %s to a path relative to %s", path, esrc) + } + mtime := info.ModTime() + atime := mtime + times := []syscall.Timespec{ + {Sec: atime.Unix(), Nsec: atime.UnixNano() % 1000000000}, + {Sec: mtime.Unix(), Nsec: mtime.UnixNano() % 1000000000}, + } + if info.IsDir() { + return addHelperDirectory(esrc, path, filepath.Join(dest, fpath), info, hostOwner, times) + } + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + return addHelperSymlink(path, filepath.Join(dest, fpath), info, hostOwner, times) + } + if !info.Mode().IsRegular() { + return errors.Errorf("error copying %q to %q: source is not a regular file; file mode is %s", path, dest, info.Mode()) + } if err = copyFileWithTar(path, filepath.Join(dest, fpath)); err != nil { return errors.Wrapf(err, "error copying %q to %q", path, dest) } @@ -343,7 +359,41 @@ func addHelper(excludes []DockerIgnore, extract bool, dest string, destfi os.Fil return nil } -func getDirsInDockerignore(srcAbsPath string, excludes []DockerIgnore) (map[string]string, error) { +func addHelperDirectory(esrc, path, dest string, info os.FileInfo, hostOwner idtools.IDPair, times []syscall.Timespec) error { + if err := idtools.MkdirAllAndChownNew(dest, info.Mode().Perm(), hostOwner); err != nil { + // discard only EEXIST on the top directory, which would have been created earlier in the caller + if !os.IsExist(err) || path != esrc { + return errors.Errorf("error creating directory %q", dest) + } + } + if err := idtools.SafeLchown(dest, hostOwner.UID, hostOwner.GID); err != nil { + return errors.Wrapf(err, "error setting owner of directory %q to %d:%d", dest, hostOwner.UID, hostOwner.GID) + } + if err := system.LUtimesNano(dest, times); err != nil { + return errors.Wrapf(err, "error setting dates on directory %q", dest) + } + return nil +} + +func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPair, times []syscall.Timespec) error { + linkContents, err := os.Readlink(src) + if err != nil { + return errors.Wrapf(err, "error reading contents of symbolic link at %q", src) + } + if err = os.Symlink(linkContents, dest); err != nil { + return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + } + if err = idtools.SafeLchown(dest, hostOwner.UID, hostOwner.GID); err != nil { + return errors.Wrapf(err, "error setting owner of symbolic link %q to %d:%d", dest, hostOwner.UID, hostOwner.GID) + } + if err = system.LUtimesNano(dest, times); err != nil { + return errors.Wrapf(err, "error setting dates on symbolic link %q", dest) + } + logrus.Debugf("Symlink(%s, %s)", linkContents, dest) + return nil +} + +func getDirsInDockerignore(srcAbsPath string, excludes []dockerIgnore) (map[string]string, error) { visitedDir := make(map[string]string) if len(excludes) == 0 { return visitedDir, nil diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 16f1a64fe..33b7afccd 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -26,7 +26,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.8.2" + Version = "1.9.0-dev" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to @@ -191,6 +191,8 @@ type Builder struct { TopLayer string // Format for the build Image Format string + // TempVolumes are temporary mount points created during container runs + TempVolumes map[string]bool } // BuilderInfo are used as objects to display container information diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index 1c3ac65f3..c65926c8e 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -1071,7 +1071,7 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( } } // Skip anything that isn't a bind or tmpfs mount. - if m.Type != "bind" && m.Type != "tmpfs" { + if m.Type != "bind" && m.Type != "tmpfs" && m.Type != "overlay" { logrus.Debugf("skipping mount of type %q on %q", m.Type, m.Destination) continue } @@ -1083,10 +1083,12 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( if err != nil { return undoBinds, errors.Wrapf(err, "error examining %q for mounting in mount namespace", m.Source) } + case "overlay": + fallthrough case "tmpfs": srcinfo, err = os.Stat("/") if err != nil { - return undoBinds, errors.Wrapf(err, "error examining / to use as a template for a tmpfs") + return undoBinds, errors.Wrapf(err, "error examining / to use as a template for a %s", m.Type) } } target := filepath.Join(spec.Root.Path, m.Destination) @@ -1145,6 +1147,12 @@ func setupChrootBindMounts(spec *specs.Spec, bundlePath string) (undoBinds func( return undoBinds, errors.Wrapf(err, "error mounting tmpfs to %q in mount namespace (%q, %q)", m.Destination, target, strings.Join(m.Options, ",")) } logrus.Debugf("mounted a tmpfs to %q", target) + case "overlay": + // Mount a overlay. + if err := mount.Mount(m.Source, target, m.Type, strings.Join(append(m.Options, "private"), ",")); err != nil { + return undoBinds, errors.Wrapf(err, "error mounting overlay to %q in mount namespace (%q, %q)", m.Destination, target, strings.Join(m.Options, ",")) + } + logrus.Debugf("mounted a overlay to %q", target) } if err = unix.Statfs(target, &fs); err != nil { return undoBinds, errors.Wrapf(err, "error checking if directory %q was bound read-only", target) diff --git a/vendor/github.com/containers/buildah/commit.go b/vendor/github.com/containers/buildah/commit.go index 05d1550b3..7a373ea5e 100644 --- a/vendor/github.com/containers/buildah/commit.go +++ b/vendor/github.com/containers/buildah/commit.go @@ -106,6 +106,22 @@ type PushOptions struct { Quiet bool } +var ( + // commitPolicy bypasses any signing requirements when committing containers to images + commitPolicy = &signature.Policy{ + Default: []signature.PolicyRequirement{signature.NewPRReject()}, + Transports: map[string]signature.PolicyTransportScopes{ + is.Transport.Name(): { + "": []signature.PolicyRequirement{ + signature.NewPRInsecureAcceptAnything(), + }, + }, + }, + } + // pushPolicy bypasses any signing requirements when pushing (copying) images from local storage + pushPolicy = commitPolicy +) + // Commit writes the contents of the container, along with its updated // configuration, to a new image in the specified location, and if we know how, // add any additional tags that were specified. Returns the ID of the new image @@ -141,11 +157,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options return "", nil, "", errors.Errorf("commit access to registry for %q is blocked by configuration", transports.ImageName(dest)) } - policy, err := signature.DefaultPolicy(systemContext) - if err != nil { - return imgID, nil, "", errors.Wrapf(err, "error obtaining default signature policy") - } - policyContext, err := signature.NewPolicyContext(policy) + policyContext, err := signature.NewPolicyContext(commitPolicy) if err != nil { return imgID, nil, "", errors.Wrapf(err, "error creating new signature policy context") } @@ -280,11 +292,7 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options return nil, "", errors.Errorf("push access to registry for %q is blocked by configuration", transports.ImageName(dest)) } - policy, err := signature.DefaultPolicy(systemContext) - if err != nil { - return nil, "", errors.Wrapf(err, "error obtaining default signature policy") - } - policyContext, err := signature.NewPolicyContext(policy) + policyContext, err := signature.NewPolicyContext(pushPolicy) if err != nil { return nil, "", errors.Wrapf(err, "error creating new signature policy context") } @@ -330,6 +338,5 @@ func Push(ctx context.Context, image string, dest types.ImageReference, options logrus.Warnf("error generating canonical reference with name %q and digest %s: %v", name, manifestDigest.String(), err) } } - fmt.Printf("Successfully pushed %s@%s\n", dest.StringWithinTransport(), manifestDigest.String()) return ref, manifestDigest, nil } diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index 85848e297..b8b9db0f3 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -27,6 +27,7 @@ import ( "github.com/containers/image/types" "github.com/containers/storage" "github.com/containers/storage/pkg/archive" + "github.com/cyphar/filepath-securejoin" docker "github.com/fsouza/go-dockerclient" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" @@ -480,22 +481,22 @@ func (s *StageExecutor) volumeCacheRestore() error { } // Copy copies data into the working tree. The "Download" field is how -// imagebuilder tells us the instruction was "ADD" and not "COPY". +// imagebuilder tells us the instruction was "ADD" and not "COPY" func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) error { for _, copy := range copies { - // If the file exists, check to see if it's a symlink. - // If it is a symlink, convert to it's target otherwise - // the symlink will be overwritten. - fileDest, _ := os.Lstat(filepath.Join(s.mountPoint, copy.Dest)) - if fileDest != nil { - if fileDest.Mode()&os.ModeSymlink != 0 { - if symLink, err := resolveSymlink(s.mountPoint, copy.Dest); err == nil { - copy.Dest = symLink - } else { - return errors.Wrapf(err, "error reading symbolic link to %q", copy.Dest) - } - } + // Check the file and see if part of it is a symlink. + // Convert it to the target if so. To be ultrasafe + // do the same for the mountpoint. + secureMountPoint, err := securejoin.SecureJoin("", s.mountPoint) + finalPath, err := securejoin.SecureJoin(secureMountPoint, copy.Dest) + if err != nil { + return errors.Wrapf(err, "error resolving symlinks for copy destination %s", copy.Dest) + } + if !strings.HasPrefix(finalPath, secureMountPoint) { + return errors.Wrapf(err, "error resolving copy destination %s", copy.Dest) } + copy.Dest = strings.TrimPrefix(finalPath, secureMountPoint) + if copy.Download { logrus.Debugf("ADD %#v, %#v", excludes, copy) } else { @@ -1218,22 +1219,32 @@ func (s *StageExecutor) layerExists(ctx context.Context, currNode *parser.Node, if err != nil { return "", errors.Wrap(err, "error getting image list from store") } - for _, image := range images { - layer, err := s.executor.store.Layer(image.TopLayer) + var baseHistory []v1.History + if s.builder.FromImageID != "" { + baseHistory, err = s.executor.getImageHistory(ctx, s.builder.FromImageID) if err != nil { - return "", errors.Wrapf(err, "error getting top layer info") + return "", errors.Wrapf(err, "error getting history of base image %q", s.builder.FromImageID) + } + } + for _, image := range images { + var imageTopLayer *storage.Layer + if image.TopLayer != "" { + imageTopLayer, err = s.executor.store.Layer(image.TopLayer) + if err != nil { + return "", errors.Wrapf(err, "error getting top layer info") + } } // If the parent of the top layer of an image is equal to the last entry in b.topLayers // it means that this image is potentially a cached intermediate image from a previous // build. Next we double check that the history of this image is equivalent to the previous // lines in the Dockerfile up till the point we are at in the build. - if layer.Parent == s.executor.topLayers[len(s.executor.topLayers)-1] || layer.ID == s.executor.topLayers[len(s.executor.topLayers)-1] { + if imageTopLayer == nil || imageTopLayer.Parent == s.executor.topLayers[len(s.executor.topLayers)-1] || imageTopLayer.ID == s.executor.topLayers[len(s.executor.topLayers)-1] { history, err := s.executor.getImageHistory(ctx, image.ID) if err != nil { return "", errors.Wrapf(err, "error getting history of %q", image.ID) } // children + currNode is the point of the Dockerfile we are currently at. - if s.executor.historyMatches(append(children, currNode), history) { + if s.executor.historyMatches(baseHistory, currNode, history) { // This checks if the files copied during build have been changed if the node is // a COPY or ADD command. filesMatch, err := s.copiedFilesMatch(currNode, history[len(history)-1].Created) @@ -1282,32 +1293,60 @@ func (b *Executor) getCreatedBy(node *parser.Node) string { return "/bin/sh -c #(nop) " + node.Original } -// historyMatches returns true if the history of the image matches the lines -// in the Dockerfile till the point of build we are at. +// historyMatches returns true if a candidate history matches the history of our +// base image (if we have one), plus the current instruction. // Used to verify whether a cache of the intermediate image exists and whether // to run the build again. -func (b *Executor) historyMatches(children []*parser.Node, history []v1.History) bool { - i := len(history) - 1 - for j := len(children) - 1; j >= 0; j-- { - instruction := children[j].Original - if children[j].Value == "run" { - instruction = instruction[4:] - buildArgs := b.getBuildArgs() - // If a previous image was built with some build-args but the new build process doesn't have any build-args - // specified, so compare the lengths of the old instruction with the current one - // 11 is the length of "/bin/sh -c " that is used to run the run commands - if buildArgs == "" && len(history[i].CreatedBy) > len(instruction)+11 { - return false - } - // There are build-args, so check if anything with the build-args has changed - if buildArgs != "" && !strings.Contains(history[i].CreatedBy, buildArgs) { - return false - } +func (b *Executor) historyMatches(baseHistory []v1.History, child *parser.Node, history []v1.History) bool { + if len(baseHistory) >= len(history) { + return false + } + if len(history)-len(baseHistory) != 1 { + return false + } + for i := range baseHistory { + if baseHistory[i].CreatedBy != history[i].CreatedBy { + return false + } + if baseHistory[i].Comment != history[i].Comment { + return false + } + if baseHistory[i].Author != history[i].Author { + return false + } + if baseHistory[i].EmptyLayer != history[i].EmptyLayer { + return false + } + if baseHistory[i].Created != nil && history[i].Created == nil { + return false + } + if baseHistory[i].Created == nil && history[i].Created != nil { + return false } - if !strings.Contains(history[i].CreatedBy, instruction) { + if baseHistory[i].Created != nil && history[i].Created != nil && *baseHistory[i].Created != *history[i].Created { + return false + } + } + instruction := child.Original + switch strings.ToUpper(child.Value) { + case "RUN": + instruction = instruction[4:] + buildArgs := b.getBuildArgs() + // If a previous image was built with some build-args but the new build process doesn't have any build-args + // specified, the command might be expanded differently, so compare the lengths of the old instruction with + // the current one. 11 is the length of "/bin/sh -c " that is used to run the run commands. + if buildArgs == "" && len(history[len(baseHistory)].CreatedBy) > len(instruction)+11 { + return false + } + // There are build-args, so check if anything with the build-args has changed + if buildArgs != "" && !strings.Contains(history[len(baseHistory)].CreatedBy, buildArgs) { + return false + } + fallthrough + default: + if !strings.Contains(history[len(baseHistory)].CreatedBy, instruction) { return false } - i-- } return true } @@ -1816,8 +1855,8 @@ func (b *Executor) deleteSuccessfulIntermediateCtrs() error { } func (s *StageExecutor) EnsureContainerPath(path string) error { - targetPath := filepath.Join(s.mountPoint, path) - _, err := os.Lstat(targetPath) + targetPath, err := securejoin.SecureJoin(s.mountPoint, path) + _, err = os.Lstat(targetPath) if err != nil && os.IsNotExist(err) { err = os.MkdirAll(targetPath, 0755) } diff --git a/vendor/github.com/containers/buildah/new.go b/vendor/github.com/containers/buildah/new.go index 29546caba..ecd1666bf 100644 --- a/vendor/github.com/containers/buildah/new.go +++ b/vendor/github.com/containers/buildah/new.go @@ -351,6 +351,7 @@ func newBuilder(ctx context.Context, store storage.Store, options BuilderOptions TopLayer: topLayer, Args: options.Args, Format: options.Format, + TempVolumes: map[string]bool{}, } if options.Mount { diff --git a/vendor/github.com/containers/buildah/pkg/cli/common.go b/vendor/github.com/containers/buildah/pkg/cli/common.go index e7a571db6..23bb696fc 100644 --- a/vendor/github.com/containers/buildah/pkg/cli/common.go +++ b/vendor/github.com/containers/buildah/pkg/cli/common.go @@ -7,6 +7,7 @@ package cli import ( "fmt" "os" + "path/filepath" "strings" "github.com/containers/buildah" @@ -137,7 +138,7 @@ func GetLayerFlags(flags *LayerResults) pflag.FlagSet { func GetBudFlags(flags *BudResults) pflag.FlagSet { fs := pflag.FlagSet{} fs.StringArrayVar(&flags.Annotation, "annotation", []string{}, "Set metadata for an image (default [])") - fs.StringVar(&flags.Authfile, "authfile", "", "path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json") + fs.StringVar(&flags.Authfile, "authfile", GetDefaultAuthFile(), "path of the authentication file.") fs.StringArrayVar(&flags.BuildArg, "build-arg", []string{}, "`argument=value` to supply to the builder") fs.StringVar(&flags.CacheFrom, "cache-from", "", "Images to utilise as potential cache sources. The build process does not currently support caching so this is a NOOP.") fs.StringVar(&flags.CertDir, "cert-dir", "", "use certificates at the specified path to access the registry") @@ -246,3 +247,15 @@ func VerifyFlagsArgsOrder(args []string) error { } return nil } + +func GetDefaultAuthFile() string { + authfile := os.Getenv("REGISTRY_AUTH_FILE") + if authfile != "" { + return authfile + } + runtimeDir := os.Getenv("XDG_RUNTIME_DIR") + if runtimeDir != "" { + return filepath.Join(runtimeDir, "containers/auth.json") + } + return "" +} diff --git a/vendor/github.com/containers/buildah/pkg/overlay/overlay.go b/vendor/github.com/containers/buildah/pkg/overlay/overlay.go new file mode 100644 index 000000000..31f0c2cec --- /dev/null +++ b/vendor/github.com/containers/buildah/pkg/overlay/overlay.go @@ -0,0 +1,46 @@ +package overlay + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/containers/storage" + "github.com/containers/storage/pkg/idtools" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +// MountTemp creates a subdir of the contentDir based on the source directory +// from the source system. It then mounds up the source directory on to the +// generated mount point and returns the mount point to the caller. +func MountTemp(store storage.Store, containerId, source, dest string, rootUID, rootGID int) (specs.Mount, string, error) { + mount := specs.Mount{} + + contentDir, err := store.ContainerDirectory(containerId) + if err != nil { + return mount, "", err + } + upperDir := filepath.Join(contentDir, "upper") + workDir := filepath.Join(contentDir, "work") + if err := idtools.MkdirAllAs(upperDir, 0700, rootUID, rootGID); err != nil { + return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", upperDir) + } + if err := idtools.MkdirAllAs(workDir, 0700, rootUID, rootGID); err != nil { + return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", workDir) + } + + mount.Source = "overlay" + mount.Destination = dest + mount.Type = "overlay" + mount.Options = strings.Split(fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", source, upperDir, workDir), ",") + + return mount, contentDir, nil +} + +// RemoveTemp removes temporary mountpoint and all content from its parent +// directory +func RemoveTemp(contentDir string) error { + return os.RemoveAll(contentDir) +} diff --git a/vendor/github.com/containers/buildah/pkg/parse/parse.go b/vendor/github.com/containers/buildah/pkg/parse/parse.go index 070f4d04e..6c58f1194 100644 --- a/vendor/github.com/containers/buildah/pkg/parse/parse.go +++ b/vendor/github.com/containers/buildah/pkg/parse/parse.go @@ -14,6 +14,7 @@ import ( "unicode" "github.com/containers/buildah" + "github.com/containers/buildah/pkg/unshare" "github.com/containers/image/types" "github.com/containers/storage/pkg/idtools" "github.com/docker/go-units" @@ -155,16 +156,16 @@ func ParseVolume(volume string) (specs.Mount, error) { if len(arr) < 2 { return mount, errors.Errorf("incorrect volume format %q, should be host-dir:ctr-dir[:option]", volume) } - if err := validateVolumeHostDir(arr[0]); err != nil { + if err := ValidateVolumeHostDir(arr[0]); err != nil { return mount, err } - if err := validateVolumeCtrDir(arr[1]); err != nil { + if err := ValidateVolumeCtrDir(arr[1]); err != nil { return mount, err } mountOptions := "" if len(arr) > 2 { mountOptions = arr[2] - if err := validateVolumeOpts(arr[2]); err != nil { + if err := ValidateVolumeOpts(arr[2]); err != nil { return mount, err } } @@ -189,7 +190,7 @@ func ParseVolumes(volumes []string) error { return nil } -func validateVolumeHostDir(hostDir string) error { +func ValidateVolumeHostDir(hostDir string) error { if !filepath.IsAbs(hostDir) { return errors.Errorf("invalid host path, must be an absolute path %q", hostDir) } @@ -199,14 +200,14 @@ func validateVolumeHostDir(hostDir string) error { return nil } -func validateVolumeCtrDir(ctrDir string) error { +func ValidateVolumeCtrDir(ctrDir string) error { if !filepath.IsAbs(ctrDir) { return errors.Errorf("invalid container path, must be an absolute path %q", ctrDir) } return nil } -func validateVolumeOpts(option string) error { +func ValidateVolumeOpts(option string) error { var foundRootPropagation, foundRWRO, foundLabelChange int options := strings.Split(option, ",") for _, opt := range options { @@ -216,9 +217,12 @@ func validateVolumeOpts(option string) error { return errors.Errorf("invalid options %q, can only specify 1 'rw' or 'ro' option", option) } foundRWRO++ - case "z", "Z": + case "z", "Z", "O": + if opt == "O" && unshare.IsRootless() { + return errors.Errorf("invalid options %q, overlay mounts not supported in rootless mode", option) + } if foundLabelChange > 1 { - return errors.Errorf("invalid options %q, can only specify 1 'z' or 'Z' option", option) + return errors.Errorf("invalid options %q, can only specify 1 'z', 'Z', or 'O' option", option) } foundLabelChange++ case "private", "rprivate", "shared", "rshared", "slave", "rslave", "unbindable", "runbindable": diff --git a/vendor/github.com/containers/buildah/pull.go b/vendor/github.com/containers/buildah/pull.go index 5eec1b3dd..66e573fa7 100644 --- a/vendor/github.com/containers/buildah/pull.go +++ b/vendor/github.com/containers/buildah/pull.go @@ -2,7 +2,6 @@ package buildah import ( "context" - "fmt" "io" "strings" @@ -152,8 +151,9 @@ func localImageNameForReference(ctx context.Context, store storage.Store, srcRef return name, nil } -// Pull copies the contents of the image from somewhere else to local storage. -func Pull(ctx context.Context, imageName string, options PullOptions) error { +// Pull copies the contents of the image from somewhere else to local storage. Returns the +// ID of the local image or an error. +func Pull(ctx context.Context, imageName string, options PullOptions) (imageID string, err error) { systemContext := getSystemContext(options.Store, options.SystemContext, options.SignaturePolicyPath) boptions := BuilderOptions{ @@ -166,23 +166,23 @@ func Pull(ctx context.Context, imageName string, options PullOptions) error { storageRef, transport, img, err := resolveImage(ctx, systemContext, options.Store, boptions) if err != nil { - return err + return "", err } var errs *multierror.Error if options.AllTags { if transport != util.DefaultTransport { - return errors.New("Non-docker transport is not supported, for --all-tags pulling") + return "", errors.New("Non-docker transport is not supported, for --all-tags pulling") } repo := reference.TrimNamed(storageRef.DockerReference()) dockerRef, err := docker.NewReference(reference.TagNameOnly(storageRef.DockerReference())) if err != nil { - return errors.Wrapf(err, "internal error creating docker.Transport reference for %s", storageRef.DockerReference().String()) + return "", errors.Wrapf(err, "internal error creating docker.Transport reference for %s", storageRef.DockerReference().String()) } tags, err := docker.GetRepositoryTags(ctx, systemContext, dockerRef) if err != nil { - return errors.Wrapf(err, "error getting repository tags") + return "", errors.Wrapf(err, "error getting repository tags") } for _, tag := range tags { tagged, err := reference.WithTag(repo, tag) @@ -192,7 +192,7 @@ func Pull(ctx context.Context, imageName string, options PullOptions) error { } taggedRef, err := docker.NewReference(tagged) if err != nil { - return errors.Wrapf(err, "internal error creating docker.Transport reference for %s", tagged.String()) + return "", errors.Wrapf(err, "internal error creating docker.Transport reference for %s", tagged.String()) } if options.ReportWriter != nil { options.ReportWriter.Write([]byte("Pulling " + tagged.String() + "\n")) @@ -207,13 +207,13 @@ func Pull(ctx context.Context, imageName string, options PullOptions) error { errs = multierror.Append(errs, err) continue } - fmt.Printf("%s\n", taggedImg.ID) + imageID = taggedImg.ID } } else { - fmt.Printf("%s\n", img.ID) + imageID = img.ID } - return errs.ErrorOrNil() + return imageID, errs.ErrorOrNil() } func pullImage(ctx context.Context, store storage.Store, srcRef types.ImageReference, options PullOptions, sc *types.SystemContext) (types.ImageReference, error) { diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 1acf655eb..81ce2b944 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -23,6 +23,7 @@ import ( "github.com/containernetworking/cni/libcni" "github.com/containers/buildah/bind" "github.com/containers/buildah/chroot" + "github.com/containers/buildah/pkg/overlay" "github.com/containers/buildah/pkg/secrets" "github.com/containers/buildah/pkg/unshare" "github.com/containers/buildah/util" @@ -184,6 +185,7 @@ func (b *Builder) Run(command []string, options RunOptions) error { if err != nil { return errors.Wrapf(err, "error resolving mountpoints for container %q", b.ContainerID) } + defer b.cleanupTempVolumes() if options.CNIConfigDir == "" { options.CNIConfigDir = b.CNIConfigDir @@ -214,7 +216,7 @@ func (b *Builder) Run(command []string, options RunOptions) error { if options.NoPivot { moreCreateArgs = append(moreCreateArgs, "--no-pivot") } - if err := setupRootlessSpecChanges(spec, path, rootUID, rootGID); err != nil { + if err := setupRootlessSpecChanges(spec, path, rootUID, rootGID, b.CommonBuildOpts.ShmSize); err != nil { return err } err = b.runUsingRuntimeSubproc(isolation, options, configureNetwork, configureNetworks, moreCreateArgs, spec, mountPoint, path, Package+"-"+filepath.Base(path)) @@ -438,7 +440,7 @@ func (b *Builder) setupMounts(mountPoint string, spec *specs.Spec, bundlePath st } // Get the list of explicitly-specified volume mounts. - volumes, err := runSetupVolumeMounts(spec.Linux.MountLabel, volumeMounts, optionMounts) + volumes, err := b.runSetupVolumeMounts(spec.Linux.MountLabel, volumeMounts, optionMounts, int(rootUID), int(rootGID)) if err != nil { return err } @@ -1537,11 +1539,21 @@ func addRlimits(ulimit []string, g *generate.Generator) error { return nil } -func runSetupVolumeMounts(mountLabel string, volumeMounts []string, optionMounts []specs.Mount) ([]specs.Mount, error) { - var mounts []specs.Mount +func (b *Builder) cleanupTempVolumes() { + for tempVolume, val := range b.TempVolumes { + if val { + if err := overlay.RemoveTemp(tempVolume); err != nil { + logrus.Errorf(err.Error()) + } + b.TempVolumes[tempVolume] = false + } + } +} + +func (b *Builder) runSetupVolumeMounts(mountLabel string, volumeMounts []string, optionMounts []specs.Mount, rootUID, rootGID int) (mounts []specs.Mount, Err error) { parseMount := func(host, container string, options []string) (specs.Mount, error) { - var foundrw, foundro, foundz, foundZ bool + var foundrw, foundro, foundz, foundZ, foundO bool var rootProp string for _, opt := range options { switch opt { @@ -1553,6 +1565,8 @@ func runSetupVolumeMounts(mountLabel string, volumeMounts []string, optionMounts foundz = true case "Z": foundZ = true + case "O": + foundO = true case "private", "rprivate", "slave", "rslave", "shared", "rshared": rootProp = opt } @@ -1570,6 +1584,14 @@ func runSetupVolumeMounts(mountLabel string, volumeMounts []string, optionMounts return specs.Mount{}, errors.Wrapf(err, "relabeling %q failed", host) } } + if foundO { + overlayMount, contentDir, err := overlay.MountTemp(b.store, b.ContainerID, host, container, rootUID, rootGID) + if err == nil { + + b.TempVolumes[contentDir] = true + } + return overlayMount, err + } if rootProp == "" { options = append(options, "private") } @@ -1577,13 +1599,14 @@ func runSetupVolumeMounts(mountLabel string, volumeMounts []string, optionMounts Destination: container, Type: "bind", Source: host, - Options: options, + Options: append(options, "rbind"), }, nil } + // Bind mount volumes specified for this particular Run() invocation for _, i := range optionMounts { logrus.Debugf("setting up mounted volume at %q", i.Destination) - mount, err := parseMount(i.Source, i.Destination, append(i.Options, "rbind")) + mount, err := parseMount(i.Source, i.Destination, i.Options) if err != nil { return nil, err } @@ -1809,7 +1832,7 @@ func (b *Builder) configureEnvironment(g *generate.Generator, options RunOptions } } -func setupRootlessSpecChanges(spec *specs.Spec, bundleDir string, rootUID, rootGID uint32) error { +func setupRootlessSpecChanges(spec *specs.Spec, bundleDir string, rootUID, rootGID uint32, shmSize string) error { spec.Hostname = "" spec.Process.User.AdditionalGids = nil spec.Linux.Resources = nil @@ -1843,7 +1866,7 @@ func setupRootlessSpecChanges(spec *specs.Spec, bundleDir string, rootUID, rootG Source: "shm", Destination: "/dev/shm", Type: "tmpfs", - Options: []string{"private", "nodev", "noexec", "nosuid", "mode=1777", "size=65536k"}, + Options: []string{"private", "nodev", "noexec", "nosuid", "mode=1777", fmt.Sprintf("size=%s", shmSize)}, }, { Source: "/proc", diff --git a/vendor/github.com/containers/buildah/util/util.go b/vendor/github.com/containers/buildah/util/util.go index 629d9748c..30afe8313 100644 --- a/vendor/github.com/containers/buildah/util/util.go +++ b/vendor/github.com/containers/buildah/util/util.go @@ -112,7 +112,7 @@ func ResolveName(name string, firstRegistry string, sc *types.SystemContext, sto } for _, registry := range searchRegistries { if !registry.Blocked { - registries = append(registries, registry.URL) + registries = append(registries, registry.Location) } } searchRegistriesAreEmpty := len(registries) == 0 diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index 051f98ab8..0c982626a 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,12 +3,12 @@ github.com/blang/semver v3.5.0 github.com/BurntSushi/toml v0.2.0 github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containernetworking/cni v0.7.0-rc2 -github.com/containers/image f52cf78ebfa1916da406f8b6210d8f7764ec1185 +github.com/containers/image 9467ac9cfd92c545aa389f22f27e552de053c0f2 +github.com/cyphar/filepath-securejoin v0.2.1 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 -github.com/boltdb/bolt v1.3.1 -github.com/containers/storage v1.12.6 +github.com/containers/storage v1.12.7 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 github.com/docker/docker 54dddadc7d5d89fe0be88f76979f6f6ab0dede83 github.com/docker/docker-credential-helpers v0.6.1 @@ -66,3 +66,4 @@ github.com/onsi/gomega v1.4.3 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb +github.com/etcd-io/bbolt v1.3.2 diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 40f11c62a..81a43e0fa 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -23,7 +23,7 @@ import ( "github.com/containers/image/types" "github.com/docker/distribution/registry/client" "github.com/docker/go-connections/tlsconfig" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -81,28 +81,30 @@ type bearerToken struct { // dockerClient is configuration for dealing with a single Docker registry. type dockerClient struct { // The following members are set by newDockerClient and do not change afterwards. - sys *types.SystemContext - registry string - client *http.Client - insecureSkipTLSVerify bool + sys *types.SystemContext + registry string + // tlsClientConfig is setup by newDockerClient and will be used and updated + // by detectProperties(). Callers can edit tlsClientConfig.InsecureSkipVerify in the meantime. + tlsClientConfig *tls.Config // The following members are not set by newDockerClient and must be set by callers if needed. username string password string signatureBase signatureStorageBase scope authScope + // The following members are detected registry properties: // They are set after a successful detectProperties(), and never change afterwards. - scheme string // Empty value also used to indicate detectProperties() has not yet succeeded. + client *http.Client + scheme string challenges []challenge supportsSignatures bool // Private state for setupRequestAuth (key: string, value: bearerToken) tokenCache sync.Map - // detectPropertiesError caches the initial error. - detectPropertiesError error - // detectPropertiesOnce is used to execuute detectProperties() at most once in in makeRequest(). - detectPropertiesOnce sync.Once + // Private state for detectProperties: + detectPropertiesOnce sync.Once // detectPropertiesOnce is used to execute detectProperties() at most once. + detectPropertiesError error // detectPropertiesError caches the initial error. } type authScope struct { @@ -229,8 +231,7 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc if registry == dockerHostname { registry = dockerRegistry } - tr := tlsclientconfig.NewTransport() - tr.TLSClientConfig = serverDefault() + tlsClientConfig := serverDefault() // It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry, // because docker/docker does not read the certs.d subdirectory at all in that case. We use the user-visible @@ -241,38 +242,31 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc if err != nil { return nil, err } - if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil { + if err := tlsclientconfig.SetupCertificates(certDir, tlsClientConfig); err != nil { return nil, err } // Check if TLS verification shall be skipped (default=false) which can - // either be specified in the sysregistriesv2 configuration or via the - // SystemContext, whereas the SystemContext is prioritized. + // be specified in the sysregistriesv2 configuration. skipVerify := false - if sys != nil && sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined { - // Only use the SystemContext if the actual value is defined. - skipVerify = sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue - } else { - reg, err := sysregistriesv2.FindRegistry(sys, reference) - if err != nil { - return nil, errors.Wrapf(err, "error loading registries") - } - if reg != nil { - skipVerify = reg.Insecure - } + reg, err := sysregistriesv2.FindRegistry(sys, reference) + if err != nil { + return nil, errors.Wrapf(err, "error loading registries") + } + if reg != nil { + skipVerify = reg.Insecure } - tr.TLSClientConfig.InsecureSkipVerify = skipVerify + tlsClientConfig.InsecureSkipVerify = skipVerify return &dockerClient{ - sys: sys, - registry: registry, - client: &http.Client{Transport: tr}, - insecureSkipTLSVerify: skipVerify, + sys: sys, + registry: registry, + tlsClientConfig: tlsClientConfig, }, nil } // CheckAuth validates the credentials by attempting to log into the registry -// returns an error if an error occcured while making the http request or the status code received was 401 +// returns an error if an error occurred while making the http request or the status code received was 401 func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password, registry string) error { client, err := newDockerClient(sys, registry, registry) if err != nil { @@ -557,9 +551,14 @@ func (c *dockerClient) getBearerToken(ctx context.Context, challenge challenge, // detectPropertiesHelper performs the work of detectProperties which executes // it at most once. func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { - if c.scheme != "" { - return nil + // We overwrite the TLS clients `InsecureSkipVerify` only if explicitly + // specified by the system context + if c.sys != nil && c.sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined { + c.tlsClientConfig.InsecureSkipVerify = c.sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue } + tr := tlsclientconfig.NewTransport() + tr.TLSClientConfig = c.tlsClientConfig + c.client = &http.Client{Transport: tr} ping := func(scheme string) error { url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry) @@ -579,7 +578,7 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { return nil } err := ping("https") - if err != nil && c.insecureSkipTLSVerify { + if err != nil && c.tlsClientConfig.InsecureSkipVerify { err = ping("http") } if err != nil { @@ -603,7 +602,7 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { return true } isV1 := pingV1("https") - if !isV1 && c.insecureSkipTLSVerify { + if !isV1 && c.tlsClientConfig.InsecureSkipVerify { isV1 = pingV1("http") } if isV1 { diff --git a/vendor/github.com/containers/image/docker/docker_image.go b/vendor/github.com/containers/image/docker/docker_image.go index 530c7513e..744667f54 100644 --- a/vendor/github.com/containers/image/docker/docker_image.go +++ b/vendor/github.com/containers/image/docker/docker_image.go @@ -25,7 +25,7 @@ type Image struct { // a client to the registry hosting the given image. // The caller must call .Close() on the returned Image. func newImage(ctx context.Context, sys *types.SystemContext, ref dockerReference) (types.ImageCloser, error) { - s, err := newImageSource(sys, ref) + s, err := newImageSource(ctx, sys, ref) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 8367792bf..c8fdb407c 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -13,9 +13,10 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/manifest" + "github.com/containers/image/pkg/sysregistriesv2" "github.com/containers/image/types" "github.com/docker/distribution/registry/client" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -28,17 +29,94 @@ type dockerImageSource struct { cachedManifestMIMEType string // Only valid if cachedManifest != nil } -// newImageSource creates a new ImageSource for the specified image reference. -// The caller must call .Close() on the returned ImageSource. -func newImageSource(sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) { - c, err := newDockerClientFromRef(sys, ref, false, "pull") +// newImageSource creates a new `ImageSource` for the specified image reference +// `ref`. +// +// The following steps will be done during the instance creation: +// +// - Lookup the registry within the configured location in +// `sys.SystemRegistriesConfPath`. If there is no configured registry available, +// we fallback to the provided docker reference `ref`. +// +// - References which contain a configured prefix will be automatically rewritten +// to the correct target reference. For example, if the configured +// `prefix = "example.com/foo"`, `location = "example.com"` and the image will be +// pulled from the ref `example.com/foo/image`, then the resulting pull will +// effectively point to `example.com/image`. +// +// - If the rewritten reference succeeds, it will be used as the `dockerRef` +// in the client. If the rewrite fails, the function immediately returns an error. +// +// - Each mirror will be used (in the configured order) to test the +// availability of the image manifest on the remote location. For example, +// if the manifest is not reachable due to connectivity issues, then the next +// mirror will be tested instead. If no mirror is configured or contains the +// target manifest, then the initial `ref` will be tested as fallback. The +// creation of the new `dockerImageSource` only succeeds if a remote +// location with the available manifest was found. +// +// A cleanup call to `.Close()` is needed if the caller is done using the returned +// `ImageSource`. +func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) { + registry, err := sysregistriesv2.FindRegistry(sys, ref.ref.Name()) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error loading registries configuration") + } + + if registry == nil { + // No configuration was found for the provided reference, so we create + // a fallback registry by hand to make the client creation below work + // as intended. + registry = &sysregistriesv2.Registry{ + Endpoint: sysregistriesv2.Endpoint{ + Location: ref.ref.String(), + }, + Prefix: ref.ref.String(), + } + } + + primaryDomain := reference.Domain(ref.ref) + // Found the registry within the sysregistriesv2 configuration. Now we test + // all endpoints for the manifest availability. If a working image source + // was found, it will be used for all future pull actions. + manifestLoadErr := errors.New("Internal error: newImageSource returned without trying any endpoint") + for _, endpoint := range append(registry.Mirrors, registry.Endpoint) { + logrus.Debugf("Trying to pull %q from endpoint %q", ref.ref, endpoint.Location) + + newRef, err := endpoint.RewriteReference(ref.ref, registry.Prefix) + if err != nil { + return nil, err + } + dockerRef, err := newReference(newRef) + if err != nil { + return nil, err + } + + endpointSys := sys + // sys.DockerAuthConfig does not explicitly specify a registry; we must not blindly send the credentials intended for the primary endpoint to mirrors. + if endpointSys != nil && endpointSys.DockerAuthConfig != nil && reference.Domain(dockerRef.ref) != primaryDomain { + copy := *endpointSys + copy.DockerAuthConfig = nil + endpointSys = © + } + + client, err := newDockerClientFromRef(endpointSys, dockerRef, false, "pull") + if err != nil { + return nil, err + } + client.tlsClientConfig.InsecureSkipVerify = endpoint.Insecure + + testImageSource := &dockerImageSource{ + ref: dockerRef, + c: client, + } + + manifestLoadErr = testImageSource.ensureManifestIsLoaded(ctx) + if manifestLoadErr == nil { + return testImageSource, nil + } } - return &dockerImageSource{ - ref: ref, - c: c, - }, nil + return nil, manifestLoadErr } // Reference returns the reference used to set up this source, _as specified by the user_ diff --git a/vendor/github.com/containers/image/docker/docker_transport.go b/vendor/github.com/containers/image/docker/docker_transport.go index 3c67efb4a..45da7c96f 100644 --- a/vendor/github.com/containers/image/docker/docker_transport.go +++ b/vendor/github.com/containers/image/docker/docker_transport.go @@ -61,8 +61,13 @@ func ParseReference(refString string) (types.ImageReference, error) { // NewReference returns a Docker reference for a named reference. The reference must satisfy !reference.IsNameOnly(). func NewReference(ref reference.Named) (types.ImageReference, error) { + return newReference(ref) +} + +// newReference returns a dockerReference for a named reference. +func newReference(ref reference.Named) (dockerReference, error) { if reference.IsNameOnly(ref) { - return nil, errors.Errorf("Docker reference %s has neither a tag nor a digest", reference.FamiliarString(ref)) + return dockerReference{}, errors.Errorf("Docker reference %s has neither a tag nor a digest", reference.FamiliarString(ref)) } // A github.com/distribution/reference value can have a tag and a digest at the same time! // The docker/distribution API does not really support that (we can’t ask for an image with a specific @@ -72,8 +77,9 @@ func NewReference(ref reference.Named) (types.ImageReference, error) { _, isTagged := ref.(reference.NamedTagged) _, isDigested := ref.(reference.Canonical) if isTagged && isDigested { - return nil, errors.Errorf("Docker references with both a tag and digest are currently not supported") + return dockerReference{}, errors.Errorf("Docker references with both a tag and digest are currently not supported") } + return dockerReference{ ref: ref, }, nil @@ -135,7 +141,7 @@ func (ref dockerReference) NewImage(ctx context.Context, sys *types.SystemContex // NewImageSource returns a types.ImageSource for this reference. // The caller must call .Close() on the returned ImageSource. func (ref dockerReference) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { - return newImageSource(sys, ref) + return newImageSource(ctx, sys, ref) } // NewImageDestination returns a types.ImageDestination for this reference. diff --git a/vendor/github.com/containers/image/pkg/blobinfocache/boltdb/boltdb.go b/vendor/github.com/containers/image/pkg/blobinfocache/boltdb/boltdb.go index 91d4e9137..19d0a6c80 100644 --- a/vendor/github.com/containers/image/pkg/blobinfocache/boltdb/boltdb.go +++ b/vendor/github.com/containers/image/pkg/blobinfocache/boltdb/boltdb.go @@ -7,9 +7,9 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/containers/image/pkg/blobinfocache/internal/prioritize" "github.com/containers/image/types" + bolt "github.com/etcd-io/bbolt" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" ) diff --git a/vendor/github.com/containers/image/pkg/blobinfocache/default.go b/vendor/github.com/containers/image/pkg/blobinfocache/default.go index 1e6e543b2..357333215 100644 --- a/vendor/github.com/containers/image/pkg/blobinfocache/default.go +++ b/vendor/github.com/containers/image/pkg/blobinfocache/default.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "github.com/containers/image/pkg/blobinfocache/boltdb" "github.com/containers/image/pkg/blobinfocache/memory" @@ -47,9 +48,18 @@ func blobInfoCacheDir(sys *types.SystemContext, euid int) (string, error) { return filepath.Join(dataDir, "containers", "cache"), nil } +func getRootlessUID() int { + uidEnv := os.Getenv("_CONTAINERS_ROOTLESS_UID") + if uidEnv != "" { + u, _ := strconv.Atoi(uidEnv) + return u + } + return os.Geteuid() +} + // DefaultCache returns the default BlobInfoCache implementation appropriate for sys. func DefaultCache(sys *types.SystemContext) types.BlobInfoCache { - dir, err := blobInfoCacheDir(sys, os.Geteuid()) + dir, err := blobInfoCacheDir(sys, getRootlessUID()) if err != nil { logrus.Debugf("Error determining a location for %s, using a memory-only cache", blobInfoCacheFilename) return memory.New() diff --git a/vendor/github.com/containers/image/pkg/docker/config/config.go b/vendor/github.com/containers/image/pkg/docker/config/config.go index 1f576253d..57b548e26 100644 --- a/vendor/github.com/containers/image/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/pkg/docker/config/config.go @@ -85,21 +85,6 @@ func GetAuthentication(sys *types.SystemContext, registry string) (string, strin return "", "", nil } -// GetUserLoggedIn returns the username logged in to registry from either -// auth.json or XDG_RUNTIME_DIR -// Used to tell the user if someone is logged in to the registry when logging in -func GetUserLoggedIn(sys *types.SystemContext, registry string) (string, error) { - path, err := getPathToAuth(sys) - if err != nil { - return "", err - } - username, _, _ := findAuthentication(registry, path, false) - if username != "" { - return username, nil - } - return "", nil -} - // RemoveAuthentication deletes the credentials stored in auth.json func RemoveAuthentication(sys *types.SystemContext, registry string) error { return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index 3d0bb0df2..99ae65774 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -10,6 +10,10 @@ import ( "github.com/BurntSushi/toml" "github.com/containers/image/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/containers/image/docker/reference" ) // systemRegistriesConfPath is the path to the system-wide registry @@ -22,34 +26,48 @@ var systemRegistriesConfPath = builtinRegistriesConfPath // DO NOT change this, instead see systemRegistriesConfPath above. const builtinRegistriesConfPath = "/etc/containers/registries.conf" -// Mirror represents a mirror. Mirrors can be used as pull-through caches for -// registries. -type Mirror struct { - // The mirror's URL. - URL string `toml:"url"` +// Endpoint describes a remote location of a registry. +type Endpoint struct { + // The endpoint's remote location. + Location string `toml:"location"` // If true, certs verification will be skipped and HTTP (non-TLS) // connections will be allowed. Insecure bool `toml:"insecure"` } +// RewriteReference will substitute the provided reference `prefix` to the +// endpoints `location` from the `ref` and creates a new named reference from it. +// The function errors if the newly created reference is not parsable. +func (e *Endpoint) RewriteReference(ref reference.Named, prefix string) (reference.Named, error) { + refString := ref.String() + if !refMatchesPrefix(refString, prefix) { + return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString) + } + + newNamedRef := strings.Replace(refString, prefix, e.Location, 1) + newParsedRef, err := reference.ParseNamed(newNamedRef) + if err != nil { + return nil, errors.Wrapf(err, "error rewriting reference") + } + logrus.Debugf("reference rewritten from '%v' to '%v'", refString, newParsedRef.String()) + return newParsedRef, nil +} + // Registry represents a registry. type Registry struct { - // Serializable registry URL. - URL string `toml:"url"` + // A registry is an Endpoint too + Endpoint // The registry's mirrors. - Mirrors []Mirror `toml:"mirror"` + Mirrors []Endpoint `toml:"mirror"` // If true, pulling from the registry will be blocked. Blocked bool `toml:"blocked"` - // If true, certs verification will be skipped and HTTP (non-TLS) - // connections will be allowed. - Insecure bool `toml:"insecure"` // If true, the registry can be used when pulling an unqualified image. Search bool `toml:"unqualified-search"` // Prefix is used for matching images, and to translate one namespace to - // another. If `Prefix="example.com/bar"`, `URL="example.com/foo/bar"` + // another. If `Prefix="example.com/bar"`, `location="example.com/foo/bar"` // and we pull from "example.com/bar/myimage:latest", the image will // effectively be pulled from "example.com/foo/bar/myimage:latest". - // If no Prefix is specified, it defaults to the specified URL. + // If no Prefix is specified, it defaults to the specified location. Prefix string `toml:"prefix"` } @@ -84,18 +102,18 @@ func (e *InvalidRegistries) Error() string { return e.s } -// parseURL parses the input string, performs some sanity checks and returns +// parseLocation parses the input string, performs some sanity checks and returns // the sanitized input string. An error is returned if the input string is // empty or if contains an "http{s,}://" prefix. -func parseURL(input string) (string, error) { +func parseLocation(input string) (string, error) { trimmed := strings.TrimRight(input, "/") if trimmed == "" { - return "", &InvalidRegistries{s: "invalid URL: cannot be empty"} + return "", &InvalidRegistries{s: "invalid location: cannot be empty"} } if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") { - msg := fmt.Sprintf("invalid URL '%s': URI schemes are not supported", input) + msg := fmt.Sprintf("invalid location '%s': URI schemes are not supported", input) return "", &InvalidRegistries{s: msg} } @@ -111,21 +129,21 @@ func getV1Registries(config *tomlConfig) ([]Registry, error) { // to minimize behavior inconsistency and not contribute to difficult-to-reproduce situations. registryOrder := []string{} - getRegistry := func(url string) (*Registry, error) { // Note: _pointer_ to a long-lived object + getRegistry := func(location string) (*Registry, error) { // Note: _pointer_ to a long-lived object var err error - url, err = parseURL(url) + location, err = parseLocation(location) if err != nil { return nil, err } - reg, exists := regMap[url] + reg, exists := regMap[location] if !exists { reg = &Registry{ - URL: url, - Mirrors: []Mirror{}, - Prefix: url, + Endpoint: Endpoint{Location: location}, + Mirrors: []Endpoint{}, + Prefix: location, } - regMap[url] = reg - registryOrder = append(registryOrder, url) + regMap[location] = reg + registryOrder = append(registryOrder, location) } return reg, nil } @@ -155,15 +173,15 @@ func getV1Registries(config *tomlConfig) ([]Registry, error) { } registries := []Registry{} - for _, url := range registryOrder { - reg := regMap[url] + for _, location := range registryOrder { + reg := regMap[location] registries = append(registries, *reg) } return registries, nil } // postProcessRegistries checks the consistency of all registries (e.g., set -// the Prefix to URL if not set) and applies conflict checks. It returns an +// the Prefix to Location if not set) and applies conflict checks. It returns an // array of cleaned registries and error in case of conflicts. func postProcessRegistries(regs []Registry) ([]Registry, error) { var registries []Registry @@ -172,16 +190,16 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) { for _, reg := range regs { var err error - // make sure URL and Prefix are valid - reg.URL, err = parseURL(reg.URL) + // make sure Location and Prefix are valid + reg.Location, err = parseLocation(reg.Location) if err != nil { return nil, err } if reg.Prefix == "" { - reg.Prefix = reg.URL + reg.Prefix = reg.Location } else { - reg.Prefix, err = parseURL(reg.Prefix) + reg.Prefix, err = parseLocation(reg.Prefix) if err != nil { return nil, err } @@ -189,13 +207,13 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) { // make sure mirrors are valid for _, mir := range reg.Mirrors { - mir.URL, err = parseURL(mir.URL) + mir.Location, err = parseLocation(mir.Location) if err != nil { return nil, err } } registries = append(registries, reg) - regMap[reg.URL] = append(regMap[reg.URL], reg) + regMap[reg.Location] = append(regMap[reg.Location], reg) } // Given a registry can be mentioned multiple times (e.g., to have @@ -205,15 +223,15 @@ func postProcessRegistries(regs []Registry) ([]Registry, error) { // Note: we need to iterate over the registries array to ensure a // deterministic behavior which is not guaranteed by maps. for _, reg := range registries { - others, _ := regMap[reg.URL] + others, _ := regMap[reg.Location] for _, other := range others { if reg.Insecure != other.Insecure { - msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.URL) + msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'insecure' setting", reg.Location) return nil, &InvalidRegistries{s: msg} } if reg.Blocked != other.Blocked { - msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.URL) + msg := fmt.Sprintf("registry '%s' is defined multiple times with conflicting 'blocked' setting", reg.Location) return nil, &InvalidRegistries{s: msg} } } diff --git a/vendor/github.com/containers/image/storage/storage_transport.go b/vendor/github.com/containers/image/storage/storage_transport.go index 3a6be6e00..c9a05e6c0 100644 --- a/vendor/github.com/containers/image/storage/storage_transport.go +++ b/vendor/github.com/containers/image/storage/storage_transport.go @@ -4,7 +4,6 @@ package storage import ( "fmt" - "os" "path/filepath" "strings" @@ -181,7 +180,7 @@ func (s *storageTransport) GetStore() (storage.Store, error) { // Return the transport's previously-set store. If we don't have one // of those, initialize one now. if s.store == nil { - options, err := storage.DefaultStoreOptions(os.Getuid() != 0, os.Getuid()) + options, err := storage.DefaultStoreOptionsAutoDetectUID() if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/transports/alltransports/alltransports.go b/vendor/github.com/containers/image/transports/alltransports/alltransports.go index 23b2782f6..2335c567f 100644 --- a/vendor/github.com/containers/image/transports/alltransports/alltransports.go +++ b/vendor/github.com/containers/image/transports/alltransports/alltransports.go @@ -22,6 +22,7 @@ import ( // ParseImageName converts a URL-like image name to a types.ImageReference. func ParseImageName(imgName string) (types.ImageReference, error) { + // Keep this in sync with TransportFromImageName! parts := strings.SplitN(imgName, ":", 2) if len(parts) != 2 { return nil, errors.Errorf(`Invalid image name "%s", expected colon-separated transport:reference`, imgName) @@ -32,3 +33,14 @@ func ParseImageName(imgName string) (types.ImageReference, error) { } return transport.ParseReference(parts[1]) } + +// TransportFromImageName converts an URL-like name to a types.ImageTransport or nil when +// the transport is unknown or when the input is invalid. +func TransportFromImageName(imageName string) types.ImageTransport { + // Keep this in sync with ParseImageName! + parts := strings.SplitN(imageName, ":", 2) + if len(parts) == 2 { + return transports.Get(parts[0]) + } + return nil +} diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 9fdab2314..789504348 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -401,6 +401,7 @@ type ImageInspectInfo struct { } // DockerAuthConfig contains authorization information for connecting to a registry. +// the value of Username and Password can be empty for accessing the registry anonymously type DockerAuthConfig struct { Username string Password string diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index 89b29722b..438cab17a 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -1,7 +1,7 @@ github.com/containers/image github.com/sirupsen/logrus v1.0.0 -github.com/containers/storage v1.12.1 +github.com/containers/storage v1.12.2 github.com/davecgh/go-spew 346938d642f2ec3594ed81d874461961cd0faa76 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 @@ -13,7 +13,6 @@ github.com/containerd/continuity d8fb8589b0e8e85b8c8bbaa8840226d0dfeb7371 github.com/ghodss/yaml 04f313413ffd65ce25f2541bfd2b2ceec5c0908c github.com/gorilla/mux 94e7d24fd285520f3d12ae998f7fdd6b5393d453 github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade -github.com/mattn/go-runewidth 14207d285c6c197daabb5c9793d63e7af9ab2d50 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9 github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 @@ -43,7 +42,7 @@ github.com/syndtr/gocapability master github.com/Microsoft/go-winio ab35fc04b6365e8fcb18e6e9e41ea4a02b10b175 github.com/Microsoft/hcsshim eca7177590cdcbd25bbc5df27e3b693a54b53a6a github.com/ulikunitz/xz v0.5.4 -github.com/boltdb/bolt master +github.com/etcd-io/bbolt v1.3.2 github.com/klauspost/pgzip v1.2.1 github.com/klauspost/compress v1.4.1 github.com/klauspost/cpuid v1.2.0 diff --git a/vendor/github.com/containers/image/version/version.go b/vendor/github.com/containers/image/version/version.go index 9915cb2fa..184274736 100644 --- a/vendor/github.com/containers/image/version/version.go +++ b/vendor/github.com/containers/image/version/version.go @@ -4,11 +4,11 @@ import "fmt" const ( // VersionMajor is for an API incompatible changes - VersionMajor = 0 + VersionMajor = 1 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 1 + VersionMinor = 7 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 6 + VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "-dev" diff --git a/vendor/github.com/boltdb/bolt/LICENSE b/vendor/github.com/etcd-io/bbolt/LICENSE index 004e77fe5..004e77fe5 100644 --- a/vendor/github.com/boltdb/bolt/LICENSE +++ b/vendor/github.com/etcd-io/bbolt/LICENSE diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/github.com/etcd-io/bbolt/README.md index 7d43a15b2..e9989efc5 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/github.com/etcd-io/bbolt/README.md @@ -1,5 +1,18 @@ -Bolt [](https://coveralls.io/r/boltdb/bolt?branch=master) [](https://godoc.org/github.com/boltdb/bolt)  -==== +bbolt +===== + +[](https://goreportcard.com/report/github.com/etcd-io/bbolt) +[](https://codecov.io/gh/etcd-io/bbolt) +[](https://travis-ci.com/etcd-io/bbolt) +[](https://godoc.org/github.com/etcd-io/bbolt) +[](https://github.com/etcd-io/bbolt/releases) +[](https://github.com/etcd-io/bbolt/blob/master/LICENSE) + +bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value +store. The purpose of this fork is to provide the Go community with an active +maintenance and development target for Bolt; the goal is improved reliability +and stability. bbolt includes bug fixes, performance enhancements, and features +not found in Bolt while preserving backwards compatibility with the Bolt API. Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] [LMDB project][lmdb]. The goal of the project is to provide a simple, @@ -10,6 +23,8 @@ Since Bolt is meant to be used as such a low-level piece of functionality, simplicity is key. The API will be small and only focus on getting values and setting values. That's it. +[gh_ben]: https://github.com/benbjohnson +[bolt]: https://github.com/boltdb/bolt [hyc_symas]: https://twitter.com/hyc_symas [lmdb]: http://symas.com/mdb/ @@ -21,36 +36,42 @@ consistency and thread safety. Bolt is currently used in high-load production environments serving databases as large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed services every day. +## Project versioning + +bbolt uses [semantic versioning](http://semver.org). +API should not change between patch and minor releases. +New minor versions may add additional features to the API. + ## Table of Contents -- [Getting Started](#getting-started) - - [Installing](#installing) - - [Opening a database](#opening-a-database) - - [Transactions](#transactions) - - [Read-write transactions](#read-write-transactions) - - [Read-only transactions](#read-only-transactions) - - [Batch read-write transactions](#batch-read-write-transactions) - - [Managing transactions manually](#managing-transactions-manually) - - [Using buckets](#using-buckets) - - [Using key/value pairs](#using-keyvalue-pairs) - - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) - - [Iterating over keys](#iterating-over-keys) - - [Prefix scans](#prefix-scans) - - [Range scans](#range-scans) - - [ForEach()](#foreach) - - [Nested buckets](#nested-buckets) - - [Database backups](#database-backups) - - [Statistics](#statistics) - - [Read-Only Mode](#read-only-mode) - - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) -- [Resources](#resources) -- [Comparison with other databases](#comparison-with-other-databases) - - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) - - [LevelDB, RocksDB](#leveldb-rocksdb) - - [LMDB](#lmdb) -- [Caveats & Limitations](#caveats--limitations) -- [Reading the Source](#reading-the-source) -- [Other Projects Using Bolt](#other-projects-using-bolt) + - [Getting Started](#getting-started) + - [Installing](#installing) + - [Opening a database](#opening-a-database) + - [Transactions](#transactions) + - [Read-write transactions](#read-write-transactions) + - [Read-only transactions](#read-only-transactions) + - [Batch read-write transactions](#batch-read-write-transactions) + - [Managing transactions manually](#managing-transactions-manually) + - [Using buckets](#using-buckets) + - [Using key/value pairs](#using-keyvalue-pairs) + - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) + - [Iterating over keys](#iterating-over-keys) + - [Prefix scans](#prefix-scans) + - [Range scans](#range-scans) + - [ForEach()](#foreach) + - [Nested buckets](#nested-buckets) + - [Database backups](#database-backups) + - [Statistics](#statistics) + - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) + - [Resources](#resources) + - [Comparison with other databases](#comparison-with-other-databases) + - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) + - [LevelDB, RocksDB](#leveldb-rocksdb) + - [LMDB](#lmdb) + - [Caveats & Limitations](#caveats--limitations) + - [Reading the Source](#reading-the-source) + - [Other Projects Using Bolt](#other-projects-using-bolt) ## Getting Started @@ -59,13 +80,28 @@ Shopify and Heroku use Bolt-backed services every day. To start using Bolt, install Go and run `go get`: ```sh -$ go get github.com/boltdb/bolt/... +$ go get go.etcd.io/bbolt/... ``` This will retrieve the library and install the `bolt` command line utility into your `$GOBIN` path. +### Importing bbolt + +To use bbolt as an embedded key-value store, import as: + +```go +import bolt "go.etcd.io/bbolt" + +db, err := bolt.Open(path, 0666, nil) +if err != nil { + return err +} +defer db.Close() +``` + + ### Opening a database The top-level object in Bolt is a `DB`. It is represented as a single file on @@ -79,7 +115,7 @@ package main import ( "log" - "github.com/boltdb/bolt" + bolt "go.etcd.io/bbolt" ) func main() { @@ -522,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block your other database reads and writes. By default, it will use a regular file handle which will utilize the operating -system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx) +system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx) documentation for information about optimizing for larger-than-RAM datasets. One common use case is to backup over HTTP so you can use tools like `cURL` to @@ -811,7 +847,7 @@ Here are a few things to note when evaluating and using Bolt: ## Reading the Source -Bolt is a relatively small code base (<3KLOC) for an embedded, serializable, +Bolt is a relatively small code base (<5KLOC) for an embedded, serializable, transactional key/value database so it can be a good starting point for people interested in how databases work. @@ -863,54 +899,56 @@ them via pull request. Below is a list of public, open source projects that use Bolt: -* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. -* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. -* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. -* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. -* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. -* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. +* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. +* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining + simple tx and key scans. +* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. -* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. -* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. +* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. +* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. * [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. +* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. +* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. * [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. -* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. -* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. +* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies +* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. +* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding. * [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. -* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. -* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. -* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. -* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. -* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. -* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. +* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. * [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. -* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. -* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. -* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. +* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi. +* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service +* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [stow](https://github.com/djherbis/stow) - a persistence manager for objects backed by boltdb. -* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining - simple tx and key scans. -* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. -* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service -* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. -* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. -* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. * [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. -* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. -* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. -* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. -* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. +* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. +* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. * [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. -* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains -* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. -* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. -* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. -* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies -* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB -* [Ponzu CMS](https://ponzu-cms.org) - Headless CMS + automatic JSON API with auto-HTTPS, HTTP/2 Server Push, and flexible server framework. +* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/bolt_386.go b/vendor/github.com/etcd-io/bbolt/bolt_386.go index 820d533c1..4d35ee7cf 100644 --- a/vendor/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_386.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/boltdb/bolt/bolt_amd64.go b/vendor/github.com/etcd-io/bbolt/bolt_amd64.go index 98fafdb47..60a52dad5 100644 --- a/vendor/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_amd64.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_arm.go b/vendor/github.com/etcd-io/bbolt/bolt_arm.go index 7e5cb4b94..105d27ddb 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_arm.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "unsafe" diff --git a/vendor/github.com/boltdb/bolt/bolt_arm64.go b/vendor/github.com/etcd-io/bbolt/bolt_arm64.go index b26d84f91..f5aa2a5ee 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_arm64.go @@ -1,6 +1,6 @@ // +build arm64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_linux.go b/vendor/github.com/etcd-io/bbolt/bolt_linux.go index 2b6766614..7707bcacf 100644 --- a/vendor/github.com/boltdb/bolt/bolt_linux.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_linux.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/etcd-io/bbolt/bolt_mips64x.go b/vendor/github.com/etcd-io/bbolt/bolt_mips64x.go new file mode 100644 index 000000000..baeb289fd --- /dev/null +++ b/vendor/github.com/etcd-io/bbolt/bolt_mips64x.go @@ -0,0 +1,12 @@ +// +build mips64 mips64le + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x8000000000 // 512GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/etcd-io/bbolt/bolt_mipsx.go b/vendor/github.com/etcd-io/bbolt/bolt_mipsx.go new file mode 100644 index 000000000..2d9b1a91f --- /dev/null +++ b/vendor/github.com/etcd-io/bbolt/bolt_mipsx.go @@ -0,0 +1,12 @@ +// +build mips mipsle + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x40000000 // 1GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_openbsd.go b/vendor/github.com/etcd-io/bbolt/bolt_openbsd.go index 7058c3d73..d7f50358e 100644 --- a/vendor/github.com/boltdb/bolt/bolt_openbsd.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_openbsd.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc.go b/vendor/github.com/etcd-io/bbolt/bolt_ppc.go index 645ddc3ed..69804714a 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_ppc.go @@ -1,9 +1,12 @@ // +build ppc -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64.go b/vendor/github.com/etcd-io/bbolt/bolt_ppc64.go index 9331d9771..356590857 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_ppc64.go @@ -1,6 +1,6 @@ // +build ppc64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/github.com/etcd-io/bbolt/bolt_ppc64le.go index 8c143bc5d..422c7c69d 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_ppc64le.go @@ -1,6 +1,6 @@ // +build ppc64le -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_s390x.go b/vendor/github.com/etcd-io/bbolt/bolt_s390x.go index d7c39af92..6d3fcb825 100644 --- a/vendor/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_s390x.go @@ -1,6 +1,6 @@ // +build s390x -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/boltdb/bolt/bolt_unix.go b/vendor/github.com/etcd-io/bbolt/bolt_unix.go index cad62dda1..5f2bb5145 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_unix.go @@ -1,41 +1,43 @@ // +build !windows,!plan9,!solaris -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + flag := syscall.LOCK_NB + if exclusive { + flag |= syscall.LOCK_EX + } else { + flag |= syscall.LOCK_SH + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - flag := syscall.LOCK_SH - if exclusive { - flag = syscall.LOCK_EX - } - - // Otherwise attempt to obtain an exclusive lock. - err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB) + // Attempt to obtain an exclusive lock. + err := syscall.Flock(int(fd), flag) if err == nil { return nil } else if err != syscall.EWOULDBLOCK { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } @@ -53,7 +55,9 @@ func mmap(db *DB, sz int) error { } // Advise the kernel that the mmap is accessed randomly. - if err := madvise(b, syscall.MADV_RANDOM); err != nil { + err = madvise(b, syscall.MADV_RANDOM) + if err != nil && err != syscall.ENOSYS { + // Ignore not implemented error in kernel because it still works. return fmt.Errorf("madvise: %s", err) } diff --git a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go b/vendor/github.com/etcd-io/bbolt/bolt_unix_solaris.go index 307bf2b3e..babad6578 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_unix_solaris.go @@ -1,8 +1,7 @@ -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" @@ -11,36 +10,35 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + var lockType int16 + if exclusive { + lockType = syscall.F_WRLCK + } else { + lockType = syscall.F_RDLCK + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - var lock syscall.Flock_t - lock.Start = 0 - lock.Len = 0 - lock.Pid = 0 - lock.Whence = 0 - lock.Pid = 0 - if exclusive { - lock.Type = syscall.F_WRLCK - } else { - lock.Type = syscall.F_RDLCK - } - err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock) + // Attempt to obtain an exclusive lock. + lock := syscall.Flock_t{Type: lockType} + err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) if err == nil { return nil } else if err != syscall.EAGAIN { return err } + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } diff --git a/vendor/github.com/boltdb/bolt/bolt_windows.go b/vendor/github.com/etcd-io/bbolt/bolt_windows.go index b00fb0720..fca178bd2 100644 --- a/vendor/github.com/boltdb/bolt/bolt_windows.go +++ b/vendor/github.com/etcd-io/bbolt/bolt_windows.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -16,8 +16,6 @@ var ( ) const ( - lockExt = ".lock" - // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx flagLockExclusive = 2 flagLockFailImmediately = 1 @@ -48,48 +46,47 @@ func fdatasync(db *DB) error { } // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { - // Create a separate lock file on windows because a process - // cannot share an exclusive lock on the same file. This is - // needed during Tx.WriteTo(). - f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) - if err != nil { - return err - } - db.lockfile = f - +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time + if timeout != 0 { + t = time.Now() + } + var flag uint32 = flagLockFailImmediately + if exclusive { + flag |= flagLockExclusive + } for { - // If we're beyond our timeout then return an error. - // This can only occur after we've attempted a flock once. - if t.IsZero() { - t = time.Now() - } else if timeout > 0 && time.Since(t) > timeout { - return ErrTimeout - } - - var flag uint32 = flagLockFailImmediately - if exclusive { - flag |= flagLockExclusive - } + // Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range + // -1..0 as the lock on the database file. + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) - err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) if err == nil { return nil } else if err != errLockViolation { return err } + // If we timed oumercit then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + // Wait for a bit and try again. - time.Sleep(50 * time.Millisecond) + time.Sleep(flockRetryTimeout) } } // funlock releases an advisory lock on a file descriptor. func funlock(db *DB) error { - err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) - db.lockfile.Close() - os.Remove(db.path + lockExt) + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) return err } diff --git a/vendor/github.com/boltdb/bolt/boltsync_unix.go b/vendor/github.com/etcd-io/bbolt/boltsync_unix.go index f50442523..9587afefe 100644 --- a/vendor/github.com/boltdb/bolt/boltsync_unix.go +++ b/vendor/github.com/etcd-io/bbolt/boltsync_unix.go @@ -1,6 +1,6 @@ // +build !windows,!plan9,!linux,!openbsd -package bolt +package bbolt // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { diff --git a/vendor/github.com/boltdb/bolt/bucket.go b/vendor/github.com/etcd-io/bbolt/bucket.go index 0c5bf2746..84bfd4d6a 100644 --- a/vendor/github.com/boltdb/bolt/bucket.go +++ b/vendor/github.com/etcd-io/bbolt/bucket.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -14,13 +14,6 @@ const ( MaxValueSize = (1 << 31) - 2 ) -const ( - maxUint = ^uint(0) - minUint = 0 - maxInt = int(^uint(0) >> 1) - minInt = -maxInt - 1 -) - const bucketHeaderSize = int(unsafe.Sizeof(bucket{})) const ( @@ -323,7 +316,12 @@ func (b *Bucket) Delete(key []byte) error { // Move cursor to correct position. c := b.Cursor() - _, _, flags := c.seek(key) + k, _, flags := c.seek(key) + + // Return nil if the key doesn't exist. + if !bytes.Equal(key, k) { + return nil + } // Return an error if there is already existing bucket value. if (flags & bucketLeafFlag) != 0 { diff --git a/vendor/github.com/boltdb/bolt/cursor.go b/vendor/github.com/etcd-io/bbolt/cursor.go index 1be9f35e3..3000aced6 100644 --- a/vendor/github.com/boltdb/bolt/cursor.go +++ b/vendor/github.com/etcd-io/bbolt/cursor.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -157,12 +157,6 @@ func (c *Cursor) seek(seek []byte) (key []byte, value []byte, flags uint32) { // Start from root page/node and traverse to correct page. c.stack = c.stack[:0] c.search(seek, c.bucket.root) - ref := &c.stack[len(c.stack)-1] - - // If the cursor is pointing to the end of page/node then return nil. - if ref.index >= ref.count() { - return nil, nil, 0 - } // If this is a bucket then return a nil value. return c.keyValue() @@ -339,6 +333,8 @@ func (c *Cursor) nsearch(key []byte) { // keyValue returns the key and value of the current leaf element. func (c *Cursor) keyValue() ([]byte, []byte, uint32) { ref := &c.stack[len(c.stack)-1] + + // If the cursor is pointing to the end of page/node then return nil. if ref.count() == 0 || ref.index >= ref.count() { return nil, nil, 0 } diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/github.com/etcd-io/bbolt/db.go index f352ff14f..962248c99 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/github.com/etcd-io/bbolt/db.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "errors" @@ -7,8 +7,7 @@ import ( "log" "os" "runtime" - "runtime/debug" - "strings" + "sort" "sync" "time" "unsafe" @@ -23,6 +22,8 @@ const version = 2 // Represents a marker value to indicate that a file is a Bolt DB. const magic uint32 = 0xED0CDAED +const pgidNoFreelist pgid = 0xffffffffffffffff + // IgnoreNoSync specifies whether the NoSync field of a DB is ignored when // syncing changes to a file. This is required as some operating systems, // such as OpenBSD, do not have a unified buffer cache (UBC) and writes @@ -39,6 +40,19 @@ const ( // default page size for db is set to the OS page size. var defaultPageSize = os.Getpagesize() +// The time elapsed between consecutive file locking attempts. +const flockRetryTimeout = 50 * time.Millisecond + +// FreelistType is the type of the freelist backend +type FreelistType string + +const ( + // FreelistArrayType indicates backend freelist type is array + FreelistArrayType = FreelistType("array") + // FreelistMapType indicates backend freelist type is hashmap + FreelistMapType = FreelistType("hashmap") +) + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -61,6 +75,18 @@ type DB struct { // THIS IS UNSAFE. PLEASE USE WITH CAUTION. NoSync bool + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + NoFreelistSync bool + + // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures + // dramatic performance degradation if database is large and framentation in freelist is common. + // The alternative one is using hashmap, it is faster in almost all circumstances + // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe. + // The default type is array + FreelistType FreelistType + // When true, skips the truncate call when growing the database. // Setting this to true is only safe on non-ext3/ext4 systems. // Skipping truncation avoids preallocation of hard drive space and @@ -96,8 +122,7 @@ type DB struct { path string file *os.File - lockfile *os.File // windows only - dataref []byte // mmap'ed readonly, write throws SEGV + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -107,9 +132,11 @@ type DB struct { opened bool rwtx *Tx txs []*Tx - freelist *freelist stats Stats + freelist *freelist + freelistLoad sync.Once + pagePool sync.Pool batchMu sync.Mutex @@ -148,14 +175,18 @@ func (db *DB) String() string { // If the file does not exist then it will be created automatically. // Passing in nil options will cause Bolt to open the database with the default options. func Open(path string, mode os.FileMode, options *Options) (*DB, error) { - var db = &DB{opened: true} - + db := &DB{ + opened: true, + } // Set default options if no options are provided. if options == nil { options = DefaultOptions } + db.NoSync = options.NoSync db.NoGrowSync = options.NoGrowSync db.MmapFlags = options.MmapFlags + db.NoFreelistSync = options.NoFreelistSync + db.FreelistType = options.FreelistType // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize @@ -183,7 +214,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // if !options.ReadOnly. // The database file is locked using the shared lock (more than one process may // hold a lock at the same time) otherwise (options.ReadOnly is set). - if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { + if err := flock(db, !db.readOnly, options.Timeout); err != nil { _ = db.close() return nil, err } @@ -191,31 +222,41 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // Default values for test hooks db.ops.writeAt = db.file.WriteAt + if db.pageSize = options.PageSize; db.pageSize == 0 { + // Set the default page size to the OS page size. + db.pageSize = defaultPageSize + } + // Initialize the database if it doesn't exist. if info, err := db.file.Stat(); err != nil { + _ = db.close() return nil, err } else if info.Size() == 0 { // Initialize new files with meta pages. if err := db.init(); err != nil { + // clean up file descriptor on initialization fail + _ = db.close() return nil, err } } else { // Read the first meta page to determine the page size. var buf [0x1000]byte - if _, err := db.file.ReadAt(buf[:], 0); err == nil { - m := db.pageInBuffer(buf[:], 0).meta() - if err := m.validate(); err != nil { - // If we can't read the page size, we can assume it's the same - // as the OS -- since that's how the page size was chosen in the - // first place. - // - // If the first page is invalid and this OS uses a different - // page size than what the database was created with then we - // are out of luck and cannot access the database. - db.pageSize = os.Getpagesize() - } else { + // If we can't read the page size, but can read a page, assume + // it's the same as the OS or one given -- since that's how the + // page size was chosen in the first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + // + // TODO: scan for next page + if bw, err := db.file.ReadAt(buf[:], 0); err == nil && bw == len(buf) { + if m := db.pageInBuffer(buf[:], 0).meta(); m.validate() == nil { db.pageSize = int(m.pageSize) } + } else { + _ = db.close() + return nil, ErrInvalid } } @@ -232,14 +273,50 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { return nil, err } - // Read in the freelist. - db.freelist = newFreelist() - db.freelist.read(db.page(db.meta().freelist)) + if db.readOnly { + return db, nil + } + + db.loadFreelist() + + // Flush freelist when transitioning from no sync to sync so + // NoFreelistSync unaware boltdb can open the db later. + if !db.NoFreelistSync && !db.hasSyncedFreelist() { + tx, err := db.Begin(true) + if tx != nil { + err = tx.Commit() + } + if err != nil { + _ = db.close() + return nil, err + } + } // Mark the database as opened and return. return db, nil } +// loadFreelist reads the freelist if it is synced, or reconstructs it +// by scanning the DB if it is not synced. It assumes there are no +// concurrent accesses being made to the freelist. +func (db *DB) loadFreelist() { + db.freelistLoad.Do(func() { + db.freelist = newFreelist(db.FreelistType) + if !db.hasSyncedFreelist() { + // Reconstruct free list by scanning the DB. + db.freelist.readIDs(db.freepages()) + } else { + // Read free list from freelist page. + db.freelist.read(db.page(db.meta().freelist)) + } + db.stats.FreePageN = db.freelist.free_count() + }) +} + +func (db *DB) hasSyncedFreelist() bool { + return db.meta().freelist != pgidNoFreelist +} + // mmap opens the underlying memory-mapped file and initializes the meta references. // minsz is the minimum size that the new mmap can be. func (db *DB) mmap(minsz int) error { @@ -341,9 +418,6 @@ func (db *DB) mmapSize(size int) (int, error) { // init creates a new database file and initializes its meta pages. func (db *DB) init() error { - // Set the page size to the OS page size. - db.pageSize = os.Getpagesize() - // Create two meta pages on a buffer. buf := make([]byte, db.pageSize*4) for i := 0; i < 2; i++ { @@ -387,7 +461,8 @@ func (db *DB) init() error { } // Close releases all database resources. -// All transactions must be closed before closing the database. +// It will block waiting for any open transactions to finish +// before closing the database and returning. func (db *DB) Close() error { db.rwlock.Lock() defer db.rwlock.Unlock() @@ -395,8 +470,8 @@ func (db *DB) Close() error { db.metalock.Lock() defer db.metalock.Unlock() - db.mmaplock.RLock() - defer db.mmaplock.RUnlock() + db.mmaplock.Lock() + defer db.mmaplock.Unlock() return db.close() } @@ -526,21 +601,36 @@ func (db *DB) beginRWTx() (*Tx, error) { t := &Tx{writable: true} t.init(db) db.rwtx = t + db.freePages() + return t, nil +} - // Free any pages associated with closed read-only transactions. - var minid txid = 0xFFFFFFFFFFFFFFFF - for _, t := range db.txs { - if t.meta.txid < minid { - minid = t.meta.txid - } +// freePages releases any pages associated with closed read-only transactions. +func (db *DB) freePages() { + // Free all pending pages prior to earliest open transaction. + sort.Sort(txsById(db.txs)) + minid := txid(0xFFFFFFFFFFFFFFFF) + if len(db.txs) > 0 { + minid = db.txs[0].meta.txid } if minid > 0 { db.freelist.release(minid - 1) } - - return t, nil + // Release unused txid extents. + for _, t := range db.txs { + db.freelist.releaseRange(minid, t.meta.txid-1) + minid = t.meta.txid + 1 + } + db.freelist.releaseRange(minid, txid(0xFFFFFFFFFFFFFFFF)) + // Any page both allocated and freed in an extent is safe to release. } +type txsById []*Tx + +func (t txsById) Len() int { return len(t) } +func (t txsById) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t txsById) Less(i, j int) bool { return t[i].meta.txid < t[j].meta.txid } + // removeTx removes a transaction from the database. func (db *DB) removeTx(tx *Tx) { // Release the read lock on the mmap. @@ -633,11 +723,7 @@ func (db *DB) View(fn func(*Tx) error) error { return err } - if err := t.Rollback(); err != nil { - return err - } - - return nil + return t.Rollback() } // Batch calls fn as part of a batch. It behaves similar to Update, @@ -737,9 +823,7 @@ retry: // pass success, or bolt internal errors, to all callers for _, c := range b.calls { - if c.err != nil { - c.err <- err - } + c.err <- err } break retry } @@ -826,7 +910,7 @@ func (db *DB) meta() *meta { } // allocate returns a contiguous block of memory starting at a given page. -func (db *DB) allocate(count int) (*page, error) { +func (db *DB) allocate(txid txid, count int) (*page, error) { // Allocate a temporary buffer for the page. var buf []byte if count == 1 { @@ -838,7 +922,7 @@ func (db *DB) allocate(count int) (*page, error) { p.overflow = uint32(count - 1) // Use pages from the freelist if they are available. - if p.id = db.freelist.allocate(count); p.id != 0 { + if p.id = db.freelist.allocate(txid, count); p.id != 0 { return p, nil } @@ -893,6 +977,38 @@ func (db *DB) IsReadOnly() bool { return db.readOnly } +func (db *DB) freepages() []pgid { + tx, err := db.beginTx() + defer func() { + err = tx.Rollback() + if err != nil { + panic("freepages: failed to rollback tx") + } + }() + if err != nil { + panic("freepages: failed to open read only tx") + } + + reachable := make(map[pgid]*page) + nofreed := make(map[pgid]bool) + ech := make(chan error) + go func() { + for e := range ech { + panic(fmt.Sprintf("freepages: failed to get all reachable pages (%v)", e)) + } + }() + tx.checkBucket(&tx.root, reachable, nofreed, ech) + close(ech) + + var fids []pgid + for i := pgid(2); i < db.meta().pgid; i++ { + if _, ok := reachable[i]; !ok { + fids = append(fids, i) + } + } + return fids +} + // Options represents the options that can be set when opening a database. type Options struct { // Timeout is the amount of time to wait to obtain a file lock. @@ -903,6 +1019,17 @@ type Options struct { // Sets the DB.NoGrowSync flag before memory mapping the file. NoGrowSync bool + // Do not sync freelist to disk. This improves the database write performance + // under normal operation, but requires a full database re-sync during recovery. + NoFreelistSync bool + + // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures + // dramatic performance degradation if database is large and framentation in freelist is common. + // The alternative one is using hashmap, it is faster in almost all circumstances + // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe. + // The default type is array + FreelistType FreelistType + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to // grab a shared lock (UNIX). ReadOnly bool @@ -919,13 +1046,22 @@ type Options struct { // If initialMmapSize is smaller than the previous database size, // it takes no effect. InitialMmapSize int + + // PageSize overrides the default OS page size. + PageSize int + + // NoSync sets the initial value of DB.NoSync. Normally this can just be + // set directly on the DB itself when returned from Open(), but this option + // is useful in APIs which expose Options but not the underlying DB. + NoSync bool } // DefaultOptions represent the options used if nil options are passed into Open(). // No timeout is used which will cause Bolt to wait indefinitely for a lock. var DefaultOptions = &Options{ - Timeout: 0, - NoGrowSync: false, + Timeout: 0, + NoGrowSync: false, + FreelistType: FreelistArrayType, } // Stats represents statistics about the database. @@ -960,10 +1096,6 @@ func (s *Stats) Sub(other *Stats) Stats { return diff } -func (s *Stats) add(other *Stats) { - s.TxStats.add(&other.TxStats) -} - type Info struct { Data uintptr PageSize int @@ -1002,7 +1134,8 @@ func (m *meta) copy(dest *meta) { func (m *meta) write(p *page) { if m.root.root >= m.pgid { panic(fmt.Sprintf("root bucket pgid (%d) above high water mark (%d)", m.root.root, m.pgid)) - } else if m.freelist >= m.pgid { + } else if m.freelist >= m.pgid && m.freelist != pgidNoFreelist { + // TODO: reject pgidNoFreeList if !NoFreelistSync panic(fmt.Sprintf("freelist pgid (%d) above high water mark (%d)", m.freelist, m.pgid)) } @@ -1029,11 +1162,3 @@ func _assert(condition bool, msg string, v ...interface{}) { panic(fmt.Sprintf("assertion failed: "+msg, v...)) } } - -func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } -func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } - -func printstack() { - stack := strings.Join(strings.Split(string(debug.Stack()), "\n")[2:], "\n") - fmt.Fprintln(os.Stderr, stack) -} diff --git a/vendor/github.com/boltdb/bolt/doc.go b/vendor/github.com/etcd-io/bbolt/doc.go index cc937845d..95f25f01c 100644 --- a/vendor/github.com/boltdb/bolt/doc.go +++ b/vendor/github.com/etcd-io/bbolt/doc.go @@ -1,5 +1,5 @@ /* -Package bolt implements a low-level key/value store in pure Go. It supports +package bbolt implements a low-level key/value store in pure Go. It supports fully serializable transactions, ACID semantics, and lock-free MVCC with multiple readers and a single writer. Bolt can be used for projects that want a simple data store without the need to add large dependencies such as @@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic. */ -package bolt +package bbolt diff --git a/vendor/github.com/boltdb/bolt/errors.go b/vendor/github.com/etcd-io/bbolt/errors.go index a3620a3eb..48758ca57 100644 --- a/vendor/github.com/boltdb/bolt/errors.go +++ b/vendor/github.com/etcd-io/bbolt/errors.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "errors" diff --git a/vendor/github.com/etcd-io/bbolt/freelist.go b/vendor/github.com/etcd-io/bbolt/freelist.go new file mode 100644 index 000000000..93fd85d50 --- /dev/null +++ b/vendor/github.com/etcd-io/bbolt/freelist.go @@ -0,0 +1,370 @@ +package bbolt + +import ( + "fmt" + "sort" + "unsafe" +) + +// txPending holds a list of pgids and corresponding allocation txns +// that are pending to be freed. +type txPending struct { + ids []pgid + alloctx []txid // txids allocating the ids + lastReleaseBegin txid // beginning txid of last matching releaseRange +} + +// pidSet holds the set of starting pgids which have the same span size +type pidSet map[pgid]struct{} + +// freelist represents a list of all pages that are available for allocation. +// It also tracks pages that have been freed but are still in use by open transactions. +type freelist struct { + freelistType FreelistType // freelist type + ids []pgid // all free and available free page ids. + allocs map[pgid]txid // mapping of txid that allocated a pgid. + pending map[txid]*txPending // mapping of soon-to-be free page ids by tx. + cache map[pgid]bool // fast lookup of all free and pending page ids. + freemaps map[uint64]pidSet // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size + forwardMap map[pgid]uint64 // key is start pgid, value is its span size + backwardMap map[pgid]uint64 // key is end pgid, value is its span size + allocate func(txid txid, n int) pgid // the freelist allocate func + free_count func() int // the function which gives you free page number + mergeSpans func(ids pgids) // the mergeSpan func + getFreePageIDs func() []pgid // get free pgids func + readIDs func(pgids []pgid) // readIDs func reads list of pages and init the freelist +} + +// newFreelist returns an empty, initialized freelist. +func newFreelist(freelistType FreelistType) *freelist { + f := &freelist{ + freelistType: freelistType, + allocs: make(map[pgid]txid), + pending: make(map[txid]*txPending), + cache: make(map[pgid]bool), + freemaps: make(map[uint64]pidSet), + forwardMap: make(map[pgid]uint64), + backwardMap: make(map[pgid]uint64), + } + + if freelistType == FreelistMapType { + f.allocate = f.hashmapAllocate + f.free_count = f.hashmapFreeCount + f.mergeSpans = f.hashmapMergeSpans + f.getFreePageIDs = f.hashmapGetFreePageIDs + f.readIDs = f.hashmapReadIDs + } else { + f.allocate = f.arrayAllocate + f.free_count = f.arrayFreeCount + f.mergeSpans = f.arrayMergeSpans + f.getFreePageIDs = f.arrayGetFreePageIDs + f.readIDs = f.arrayReadIDs + } + + return f +} + +// size returns the size of the page after serialization. +func (f *freelist) size() int { + n := f.count() + if n >= 0xFFFF { + // The first element will be used to store the count. See freelist.write. + n++ + } + return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n) +} + +// count returns count of pages on the freelist +func (f *freelist) count() int { + return f.free_count() + f.pending_count() +} + +// arrayFreeCount returns count of free pages(array version) +func (f *freelist) arrayFreeCount() int { + return len(f.ids) +} + +// pending_count returns count of pending pages +func (f *freelist) pending_count() int { + var count int + for _, txp := range f.pending { + count += len(txp.ids) + } + return count +} + +// copyall copies into dst a list of all free ids and all pending ids in one sorted list. +// f.count returns the minimum length required for dst. +func (f *freelist) copyall(dst []pgid) { + m := make(pgids, 0, f.pending_count()) + for _, txp := range f.pending { + m = append(m, txp.ids...) + } + sort.Sort(m) + mergepgids(dst, f.getFreePageIDs(), m) +} + +// arrayAllocate returns the starting page id of a contiguous list of pages of a given size. +// If a contiguous block cannot be found then 0 is returned. +func (f *freelist) arrayAllocate(txid txid, n int) pgid { + if len(f.ids) == 0 { + return 0 + } + + var initial, previd pgid + for i, id := range f.ids { + if id <= 1 { + panic(fmt.Sprintf("invalid page allocation: %d", id)) + } + + // Reset initial page if this is not contiguous. + if previd == 0 || id-previd != 1 { + initial = id + } + + // If we found a contiguous block then remove it and return it. + if (id-initial)+1 == pgid(n) { + // If we're allocating off the beginning then take the fast path + // and just adjust the existing slice. This will use extra memory + // temporarily but the append() in free() will realloc the slice + // as is necessary. + if (i + 1) == n { + f.ids = f.ids[i+1:] + } else { + copy(f.ids[i-n+1:], f.ids[i+1:]) + f.ids = f.ids[:len(f.ids)-n] + } + + // Remove from the free cache. + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, initial+i) + } + f.allocs[initial] = txid + return initial + } + + previd = id + } + return 0 +} + +// free releases a page and its overflow for a given transaction id. +// If the page is already free then a panic will occur. +func (f *freelist) free(txid txid, p *page) { + if p.id <= 1 { + panic(fmt.Sprintf("cannot free page 0 or 1: %d", p.id)) + } + + // Free page and all its overflow pages. + txp := f.pending[txid] + if txp == nil { + txp = &txPending{} + f.pending[txid] = txp + } + allocTxid, ok := f.allocs[p.id] + if ok { + delete(f.allocs, p.id) + } else if (p.flags & freelistPageFlag) != 0 { + // Freelist is always allocated by prior tx. + allocTxid = txid - 1 + } + + for id := p.id; id <= p.id+pgid(p.overflow); id++ { + // Verify that page is not already free. + if f.cache[id] { + panic(fmt.Sprintf("page %d already freed", id)) + } + // Add to the freelist and cache. + txp.ids = append(txp.ids, id) + txp.alloctx = append(txp.alloctx, allocTxid) + f.cache[id] = true + } +} + +// release moves all page ids for a transaction id (or older) to the freelist. +func (f *freelist) release(txid txid) { + m := make(pgids, 0) + for tid, txp := range f.pending { + if tid <= txid { + // Move transaction's pending pages to the available freelist. + // Don't remove from the cache since the page is still free. + m = append(m, txp.ids...) + delete(f.pending, tid) + } + } + f.mergeSpans(m) +} + +// releaseRange moves pending pages allocated within an extent [begin,end] to the free list. +func (f *freelist) releaseRange(begin, end txid) { + if begin > end { + return + } + var m pgids + for tid, txp := range f.pending { + if tid < begin || tid > end { + continue + } + // Don't recompute freed pages if ranges haven't updated. + if txp.lastReleaseBegin == begin { + continue + } + for i := 0; i < len(txp.ids); i++ { + if atx := txp.alloctx[i]; atx < begin || atx > end { + continue + } + m = append(m, txp.ids[i]) + txp.ids[i] = txp.ids[len(txp.ids)-1] + txp.ids = txp.ids[:len(txp.ids)-1] + txp.alloctx[i] = txp.alloctx[len(txp.alloctx)-1] + txp.alloctx = txp.alloctx[:len(txp.alloctx)-1] + i-- + } + txp.lastReleaseBegin = begin + if len(txp.ids) == 0 { + delete(f.pending, tid) + } + } + f.mergeSpans(m) +} + +// rollback removes the pages from a given pending tx. +func (f *freelist) rollback(txid txid) { + // Remove page ids from cache. + txp := f.pending[txid] + if txp == nil { + return + } + var m pgids + for i, pgid := range txp.ids { + delete(f.cache, pgid) + tx := txp.alloctx[i] + if tx == 0 { + continue + } + if tx != txid { + // Pending free aborted; restore page back to alloc list. + f.allocs[pgid] = tx + } else { + // Freed page was allocated by this txn; OK to throw away. + m = append(m, pgid) + } + } + // Remove pages from pending list and mark as free if allocated by txid. + delete(f.pending, txid) + f.mergeSpans(m) +} + +// freed returns whether a given page is in the free list. +func (f *freelist) freed(pgid pgid) bool { + return f.cache[pgid] +} + +// read initializes the freelist from a freelist page. +func (f *freelist) read(p *page) { + if (p.flags & freelistPageFlag) == 0 { + panic(fmt.Sprintf("invalid freelist page: %d, page type is %s", p.id, p.typ())) + } + // If the page.count is at the max uint16 value (64k) then it's considered + // an overflow and the size of the freelist is stored as the first element. + idx, count := 0, int(p.count) + if count == 0xFFFF { + idx = 1 + count = int(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0]) + } + + // Copy the list of page ids from the freelist. + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count] + + // copy the ids, so we don't modify on the freelist page directly + idsCopy := make([]pgid, count) + copy(idsCopy, ids) + // Make sure they're sorted. + sort.Sort(pgids(idsCopy)) + + f.readIDs(idsCopy) + } +} + +// arrayReadIDs initializes the freelist from a given list of ids. +func (f *freelist) arrayReadIDs(ids []pgid) { + f.ids = ids + f.reindex() +} + +func (f *freelist) arrayGetFreePageIDs() []pgid { + return f.ids +} + +// write writes the page ids onto a freelist page. All free and pending ids are +// saved to disk since in the event of a program crash, all pending ids will +// become free. +func (f *freelist) write(p *page) error { + // Combine the old free pgids and pgids waiting on an open transaction. + + // Update the header flag. + p.flags |= freelistPageFlag + + // The page.count can only hold up to 64k elements so if we overflow that + // number then we handle it by putting the size in the first element. + lenids := f.count() + if lenids == 0 { + p.count = uint16(lenids) + } else if lenids < 0xFFFF { + p.count = uint16(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:]) + } else { + p.count = 0xFFFF + ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:]) + } + + return nil +} + +// reload reads the freelist from a page and filters out pending items. +func (f *freelist) reload(p *page) { + f.read(p) + + // Build a cache of only pending pages. + pcache := make(map[pgid]bool) + for _, txp := range f.pending { + for _, pendingID := range txp.ids { + pcache[pendingID] = true + } + } + + // Check each page in the freelist and build a new available freelist + // with any pages not in the pending lists. + var a []pgid + for _, id := range f.getFreePageIDs() { + if !pcache[id] { + a = append(a, id) + } + } + + f.readIDs(a) +} + +// reindex rebuilds the free cache based on available and pending free lists. +func (f *freelist) reindex() { + ids := f.getFreePageIDs() + f.cache = make(map[pgid]bool, len(ids)) + for _, id := range ids { + f.cache[id] = true + } + for _, txp := range f.pending { + for _, pendingID := range txp.ids { + f.cache[pendingID] = true + } + } +} + +// arrayMergeSpans try to merge list of pages(represented by pgids) with existing spans but using array +func (f *freelist) arrayMergeSpans(ids pgids) { + sort.Sort(ids) + f.ids = pgids(f.ids).merge(ids) +} diff --git a/vendor/github.com/etcd-io/bbolt/freelist_hmap.go b/vendor/github.com/etcd-io/bbolt/freelist_hmap.go new file mode 100644 index 000000000..6a03a6c3c --- /dev/null +++ b/vendor/github.com/etcd-io/bbolt/freelist_hmap.go @@ -0,0 +1,178 @@ +package bbolt + +import "sort" + +// hashmapFreeCount returns count of free pages(hashmap version) +func (f *freelist) hashmapFreeCount() int { + // use the forwardmap to get the total count + count := 0 + for _, size := range f.forwardMap { + count += int(size) + } + return count +} + +// hashmapAllocate serves the same purpose as arrayAllocate, but use hashmap as backend +func (f *freelist) hashmapAllocate(txid txid, n int) pgid { + if n == 0 { + return 0 + } + + // if we have a exact size match just return short path + if bm, ok := f.freemaps[uint64(n)]; ok { + for pid := range bm { + // remove the span + f.delSpan(pid, uint64(n)) + + f.allocs[pid] = txid + + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, pid+pgid(i)) + } + return pid + } + } + + // lookup the map to find larger span + for size, bm := range f.freemaps { + if size < uint64(n) { + continue + } + + for pid := range bm { + // remove the initial + f.delSpan(pid, uint64(size)) + + f.allocs[pid] = txid + + remain := size - uint64(n) + + // add remain span + f.addSpan(pid+pgid(n), remain) + + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, pid+pgid(i)) + } + return pid + } + } + + return 0 +} + +// hashmapReadIDs reads pgids as input an initial the freelist(hashmap version) +func (f *freelist) hashmapReadIDs(pgids []pgid) { + f.init(pgids) + + // Rebuild the page cache. + f.reindex() +} + +// hashmapGetFreePageIDs returns the sorted free page ids +func (f *freelist) hashmapGetFreePageIDs() []pgid { + count := f.free_count() + if count == 0 { + return nil + } + + m := make([]pgid, 0, count) + for start, size := range f.forwardMap { + for i := 0; i < int(size); i++ { + m = append(m, start+pgid(i)) + } + } + sort.Sort(pgids(m)) + + return m +} + +// hashmapMergeSpans try to merge list of pages(represented by pgids) with existing spans +func (f *freelist) hashmapMergeSpans(ids pgids) { + for _, id := range ids { + // try to see if we can merge and update + f.mergeWithExistingSpan(id) + } +} + +// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward +func (f *freelist) mergeWithExistingSpan(pid pgid) { + prev := pid - 1 + next := pid + 1 + + preSize, mergeWithPrev := f.backwardMap[prev] + nextSize, mergeWithNext := f.forwardMap[next] + newStart := pid + newSize := uint64(1) + + if mergeWithPrev { + //merge with previous span + start := prev + 1 - pgid(preSize) + f.delSpan(start, preSize) + + newStart -= pgid(preSize) + newSize += preSize + } + + if mergeWithNext { + // merge with next span + f.delSpan(next, nextSize) + newSize += nextSize + } + + f.addSpan(newStart, newSize) +} + +func (f *freelist) addSpan(start pgid, size uint64) { + f.backwardMap[start-1+pgid(size)] = size + f.forwardMap[start] = size + if _, ok := f.freemaps[size]; !ok { + f.freemaps[size] = make(map[pgid]struct{}) + } + + f.freemaps[size][start] = struct{}{} +} + +func (f *freelist) delSpan(start pgid, size uint64) { + delete(f.forwardMap, start) + delete(f.backwardMap, start+pgid(size-1)) + delete(f.freemaps[size], start) + if len(f.freemaps[size]) == 0 { + delete(f.freemaps, size) + } +} + +// initial from pgids using when use hashmap version +// pgids must be sorted +func (f *freelist) init(pgids []pgid) { + if len(pgids) == 0 { + return + } + + size := uint64(1) + start := pgids[0] + + if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) { + panic("pgids not sorted") + } + + f.freemaps = make(map[uint64]pidSet) + f.forwardMap = make(map[pgid]uint64) + f.backwardMap = make(map[pgid]uint64) + + for i := 1; i < len(pgids); i++ { + // continuous page + if pgids[i] == pgids[i-1]+1 { + size++ + } else { + f.addSpan(start, size) + + size = 1 + start = pgids[i] + } + } + + // init the tail + if size != 0 && start != 0 { + f.addSpan(start, size) + } +} diff --git a/vendor/github.com/boltdb/bolt/node.go b/vendor/github.com/etcd-io/bbolt/node.go index 159318b22..6c3fa553e 100644 --- a/vendor/github.com/boltdb/bolt/node.go +++ b/vendor/github.com/etcd-io/bbolt/node.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -365,7 +365,7 @@ func (n *node) spill() error { } // Allocate contiguous space for the node. - p, err := tx.allocate((node.size() / tx.db.pageSize) + 1) + p, err := tx.allocate((node.size() + tx.db.pageSize - 1) / tx.db.pageSize) if err != nil { return err } diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/github.com/etcd-io/bbolt/page.go index cde403ae8..bca9615f0 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/github.com/etcd-io/bbolt/page.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/github.com/etcd-io/bbolt/tx.go index 6700308a2..f50864142 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/github.com/etcd-io/bbolt/tx.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -126,10 +126,7 @@ func (tx *Tx) DeleteBucket(name []byte) error { // the error is returned to the caller. func (tx *Tx) ForEach(fn func(name []byte, b *Bucket) error) error { return tx.root.ForEach(func(k, v []byte) error { - if err := fn(k, tx.root.Bucket(k)); err != nil { - return err - } - return nil + return fn(k, tx.root.Bucket(k)) }) } @@ -169,28 +166,18 @@ func (tx *Tx) Commit() error { // Free the old root bucket. tx.meta.root.root = tx.root.root - opgid := tx.meta.pgid - - // Free the freelist and allocate new pages for it. This will overestimate - // the size of the freelist but not underestimate the size (which would be bad). - tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) - p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) - if err != nil { - tx.rollback() - return err - } - if err := tx.db.freelist.write(p); err != nil { - tx.rollback() - return err + // Free the old freelist because commit writes out a fresh freelist. + if tx.meta.freelist != pgidNoFreelist { + tx.db.freelist.free(tx.meta.txid, tx.db.page(tx.meta.freelist)) } - tx.meta.freelist = p.id - // If the high water mark has moved up then attempt to grow the database. - if tx.meta.pgid > opgid { - if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { - tx.rollback() + if !tx.db.NoFreelistSync { + err := tx.commitFreelist() + if err != nil { return err } + } else { + tx.meta.freelist = pgidNoFreelist } // Write dirty pages to disk. @@ -235,6 +222,31 @@ func (tx *Tx) Commit() error { return nil } +func (tx *Tx) commitFreelist() error { + // Allocate new pages for the new free list. This will overestimate + // the size of the freelist but not underestimate the size (which would be bad). + opgid := tx.meta.pgid + p, err := tx.allocate((tx.db.freelist.size() / tx.db.pageSize) + 1) + if err != nil { + tx.rollback() + return err + } + if err := tx.db.freelist.write(p); err != nil { + tx.rollback() + return err + } + tx.meta.freelist = p.id + // If the high water mark has moved up then attempt to grow the database. + if tx.meta.pgid > opgid { + if err := tx.db.grow(int(tx.meta.pgid+1) * tx.db.pageSize); err != nil { + tx.rollback() + return err + } + } + + return nil +} + // Rollback closes the transaction and ignores all previous updates. Read-only // transactions must be rolled back and not committed. func (tx *Tx) Rollback() error { @@ -291,7 +303,9 @@ func (tx *Tx) close() { } // Copy writes the entire database to a writer. -// This function exists for backwards compatibility. Use WriteTo() instead. +// This function exists for backwards compatibility. +// +// Deprecated; Use WriteTo() instead. func (tx *Tx) Copy(w io.Writer) error { _, err := tx.WriteTo(w) return err @@ -305,7 +319,11 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { if err != nil { return 0, err } - defer func() { _ = f.Close() }() + defer func() { + if cerr := f.Close(); err == nil { + err = cerr + } + }() // Generate a meta page. We use the same page data for both meta pages. buf := make([]byte, tx.db.pageSize) @@ -333,7 +351,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { } // Move past the meta pages in the file. - if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil { + if _, err := f.Seek(int64(tx.db.pageSize*2), io.SeekStart); err != nil { return n, fmt.Errorf("seek: %s", err) } @@ -344,7 +362,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { return n, err } - return n, f.Close() + return n, nil } // CopyFile copies the entire database to file at the given path. @@ -379,6 +397,9 @@ func (tx *Tx) Check() <-chan error { } func (tx *Tx) check(ch chan error) { + // Force loading free list if opened in ReadOnly mode. + tx.db.loadFreelist() + // Check if any pages are double freed. freed := make(map[pgid]bool) all := make([]pgid, tx.db.freelist.count()) @@ -394,8 +415,10 @@ func (tx *Tx) check(ch chan error) { reachable := make(map[pgid]*page) reachable[0] = tx.page(0) // meta0 reachable[1] = tx.page(1) // meta1 - for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { - reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + if tx.meta.freelist != pgidNoFreelist { + for i := uint32(0); i <= tx.page(tx.meta.freelist).overflow; i++ { + reachable[tx.meta.freelist+pgid(i)] = tx.page(tx.meta.freelist) + } } // Recursively check buckets. @@ -453,7 +476,7 @@ func (tx *Tx) checkBucket(b *Bucket, reachable map[pgid]*page, freed map[pgid]bo // allocate returns a contiguous block of memory starting at a given page. func (tx *Tx) allocate(count int) (*page, error) { - p, err := tx.db.allocate(count) + p, err := tx.db.allocate(tx.meta.txid, count) if err != nil { return nil, err } @@ -462,7 +485,7 @@ func (tx *Tx) allocate(count int) (*page, error) { tx.pages[p.id] = p // Update statistics. - tx.stats.PageCount++ + tx.stats.PageCount += count tx.stats.PageAlloc += count * tx.db.pageSize return p, nil |