summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.cirrus.yml37
-rw-r--r--.github/workflows/multi-arch-build.yaml212
-rw-r--r--contrib/hello/Containerfile (renamed from contrib/helloimage/Containerfile)0
-rw-r--r--contrib/hello/README.md (renamed from contrib/helloimage/README.md)0
-rw-r--r--contrib/hello/podman_hello_world.c (renamed from contrib/helloimage/podman_hello_world.c)0
-rw-r--r--contrib/systemd/system/podman.service.in1
-rw-r--r--libpod/define/version.go2
-rw-r--r--libpod/networking_linux.go11
-rw-r--r--libpod/networking_slirp4netns.go107
-rw-r--r--pkg/bindings/system/system.go1
-rw-r--r--pkg/machine/qemu/config.go19
-rw-r--r--pkg/machine/qemu/machine.go269
-rw-r--r--pkg/machine/qemu/options_darwin_arm64.go6
-rw-r--r--test/e2e/version_test.go8
-rw-r--r--test/system/500-networking.bats21
15 files changed, 337 insertions, 357 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index 4001704c4..beb871c23 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -38,7 +38,7 @@ env:
UBUNTU_NAME: "ubuntu-2110"
# Google-cloud VM Images
- IMAGE_SUFFIX: "c5814666029957120"
+ IMAGE_SUFFIX: "c6261670816251904"
FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
@@ -741,6 +741,39 @@ upgrade_test_task:
always: *logs_artifacts
+image_build_task:
+ name: "Build multi-arch $CTXDIR"
+ alias: image_build
+ # Some of these container images take > 1h to build, limit
+ # this task to a specific Cirrus-Cron entry with this name.
+ only_if: $CIRRUS_CRON == 'multiarch'
+ depends_on:
+ - ext_svc_check
+ timeout_in: 120m # emulation is sssllllooooowwww
+ gce_instance:
+ <<: *standardvm
+ image_name: build-push-${IMAGE_SUFFIX}
+ # More muscle required for parallel multi-arch build
+ type: "n2-standard-4"
+ env:
+ PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1]
+ PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4]
+ CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c]
+ CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9]
+ matrix:
+ - env:
+ CTXDIR: contrib/podmanimage/upstream
+ - env:
+ CTXDIR: contrib/podmanimage/testing
+ - env:
+ CTXDIR: contrib/podmanimage/stable
+ - env:
+ CTXDIR: contrib/hello
+ script:
+ - set -a; source /etc/automation_environment; set +a
+ - main.sh $CIRRUS_REPO_CLONE_URL $CTXDIR
+
+
# This task is critical. It updates the "last-used by" timestamp stored
# in metadata for all VM images. This mechanism functions in tandem with
# an out-of-band pruning operation to remove disused VM images.
@@ -757,6 +790,7 @@ meta_task:
${FEDORA_CACHE_IMAGE_NAME}
${PRIOR_FEDORA_CACHE_IMAGE_NAME}
${UBUNTU_CACHE_IMAGE_NAME}
+ build-push-${IMAGE_SUFFIX}
BUILDID: "${CIRRUS_BUILD_ID}"
REPOREF: "${CIRRUS_REPO_NAME}"
GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4]
@@ -799,6 +833,7 @@ success_task:
- rootless_gitlab_test
- upgrade_test
- buildah_bud_test
+ - image_build
- meta
container: *smallcontainer
env:
diff --git a/.github/workflows/multi-arch-build.yaml b/.github/workflows/multi-arch-build.yaml
deleted file mode 100644
index 1dc485d71..000000000
--- a/.github/workflows/multi-arch-build.yaml
+++ /dev/null
@@ -1,212 +0,0 @@
----
-
-# Please see contrib/<reponame>image/README.md for details on the intentions
-# of this workflow.
-#
-# BIG FAT WARNING: This workflow is duplicated across containers/skopeo,
-# containers/buildah, and containers/podman. ANY AND
-# ALL CHANGES MADE HERE MUST BE MANUALLY DUPLICATED
-# TO THE OTHER REPOS.
-
-name: build multi-arch images
-
-on:
- # Upstream tends to be very active, with many merges per day.
- # Only run this daily via cron schedule, or manually, not by branch push.
- schedule:
- - cron: '0 8 * * *'
- # allows to run this workflow manually from the Actions tab
- workflow_dispatch:
-
-permissions:
- contents: read
-
-jobs:
- multi:
- name: multi-arch image build
- env:
- REPONAME: podman # No easy way to parse this out of $GITHUB_REPOSITORY
- # Server/namespace value used to format FQIN
- REPONAME_QUAY_REGISTRY: quay.io/podman
- CONTAINERS_QUAY_REGISTRY: quay.io/containers
- # list of architectures for build
- PLATFORMS: linux/amd64,linux/s390x,linux/ppc64le,linux/arm64
- # Command to execute in container to obtain project version number
- VERSION_CMD: "podman --version"
-
- # build several images (upstream, testing, stable) in parallel
- strategy:
- # By default, failure of one matrix item cancels all others
- fail-fast: false
- matrix:
- # Builds are located under contrib/<reponame>image/<source> directory
- source:
- - upstream
- - testing
- - stable
- runs-on: ubuntu-latest
- # internal registry caches build for inspection before push
- services:
- registry:
- image: quay.io/libpod/registry:2
- ports:
- - 5000:5000
- steps:
- - name: Checkout
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-
- - name: Set up QEMU
- uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
- with:
- driver-opts: network=host
- install: true
-
- - name: Build and locally push image
- uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
- with:
- context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
- file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
- platforms: ${{ env.PLATFORMS }}
- push: true
- tags: localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
-
- # Simple verification that stable images work, and
- # also grab version number use in forming the FQIN.
- - name: amd64 container sniff test
- if: matrix.source == 'stable'
- id: sniff_test
- run: |
- podman pull --tls-verify=false \
- localhost:5000/$REPONAME/${{ matrix.source }}
- VERSION_OUTPUT=$(podman run \
- localhost:5000/$REPONAME/${{ matrix.source }} \
- $VERSION_CMD)
- echo "$VERSION_OUTPUT"
- VERSION=$(awk -r -e "/^${REPONAME} version /"'{print $3}' <<<"$VERSION_OUTPUT")
- test -n "$VERSION"
- echo "::set-output name=version::$VERSION"
-
- - name: Generate image FQIN(s) to push
- id: reponame_reg
- run: |
- if [[ "${{ matrix.source }}" == 'stable' ]]; then
- # The command version in image just built
- VERSION='v${{ steps.sniff_test.outputs.version }}'
- # workaround vim syntax-highlight bug: '
- # Push both new|updated version-tag and latest-tag FQINs
- FQIN="$REPONAME_QUAY_REGISTRY/stable:$VERSION,$REPONAME_QUAY_REGISTRY/stable:latest"
- elif [[ "${{ matrix.source }}" == 'testing' ]]; then
- # Assume some contents changed, always push latest testing.
- FQIN="$REPONAME_QUAY_REGISTRY/testing:latest"
- elif [[ "${{ matrix.source }}" == 'upstream' ]]; then
- # Assume some contents changed, always push latest upstream.
- FQIN="$REPONAME_QUAY_REGISTRY/upstream:latest"
- else
- echo "::error::Unknown matrix item '${{ matrix.source }}'"
- exit 1
- fi
- echo "::warning::Pushing $FQIN"
- echo "::set-output name=fqin::${FQIN}"
- echo '::set-output name=push::true'
-
- # This is substantially similar to the above logic,
- # but only handles $CONTAINERS_QUAY_REGISTRY for
- # the stable "latest" and named-version tagged images.
- - name: Generate containers reg. image FQIN(s)
- if: matrix.source == 'stable'
- id: containers_reg
- run: |
- VERSION='v${{ steps.sniff_test.outputs.version }}'
- # workaround vim syntax-highlight bug: '
- # Push both new|updated version-tag and latest-tag FQINs
- FQIN="$CONTAINERS_QUAY_REGISTRY/$REPONAME:$VERSION,$CONTAINERS_QUAY_REGISTRY/$REPONAME:latest"
- echo "::warning::Pushing $FQIN"
- echo "::set-output name=fqin::${FQIN}"
- echo '::set-output name=push::true'
-
- - name: Define LABELS multi-line env. var. value
- run: |
- # This is a really hacky/strange workflow idiom, required
- # for setting multi-line $LABELS value for consumption in
- # a future step. There is literally no cleaner way to do this :<
- # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#multiline-strings
- function set_labels() {
- echo 'LABELS<<DELIMITER' >> "$GITHUB_ENV"
- for line; do
- echo "$line" | tee -a "$GITHUB_ENV"
- done
- echo "DELIMITER" >> "$GITHUB_ENV"
- }
-
- declare -a lines
- lines=(\
- "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git"
- "org.opencontainers.image.revision=${GITHUB_SHA}"
- "org.opencontainers.image.created=$(date -u --iso-8601=seconds)"
- )
-
- # Only the 'stable' matrix source obtains $VERSION
- if [[ "${{ matrix.source }}" == "stable" ]]; then
- lines+=(\
- "org.opencontainers.image.version=${{ steps.sniff_test.outputs.version }}"
- )
- fi
-
- set_labels "${lines[@]}"
-
- # Separate steps to login and push for $REPONAME_QUAY_REGISTRY and
- # $CONTAINERS_QUAY_REGISTRY are required, because 2 sets of credentials
- # are used and namespaced within the registry. At the same time, reuse
- # of non-shell steps is not supported by Github Actions nor are YAML
- # anchors/aliases, nor composite actions.
-
- # Push to $REPONAME_QUAY_REGISTRY for stable, testing. and upstream
- - name: Login to ${{ env.REPONAME_QUAY_REGISTRY }}
- uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
- if: steps.reponame_reg.outputs.push == 'true'
- with:
- registry: ${{ env.REPONAME_QUAY_REGISTRY }}
- # N/B: Secrets are not passed to workflows that are triggered
- # by a pull request from a fork
- username: ${{ secrets.REPONAME_QUAY_USERNAME }}
- password: ${{ secrets.REPONAME_QUAY_PASSWORD }}
-
- - name: Push images to ${{ steps.reponame_reg.outputs.fqin }}
- uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
- if: steps.reponame_reg.outputs.push == 'true'
- with:
- cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
- cache-to: type=inline
- context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
- file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
- platforms: ${{ env.PLATFORMS }}
- push: true
- tags: ${{ steps.reponame_reg.outputs.fqin }}
- labels: |
- ${{ env.LABELS }}
-
- # Push to $CONTAINERS_QUAY_REGISTRY only stable
- - name: Login to ${{ env.CONTAINERS_QUAY_REGISTRY }}
- if: steps.containers_reg.outputs.push == 'true'
- uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
- with:
- registry: ${{ env.CONTAINERS_QUAY_REGISTRY}}
- username: ${{ secrets.CONTAINERS_QUAY_USERNAME }}
- password: ${{ secrets.CONTAINERS_QUAY_PASSWORD }}
-
- - name: Push images to ${{ steps.containers_reg.outputs.fqin }}
- if: steps.containers_reg.outputs.push == 'true'
- uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
- with:
- cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }}
- cache-to: type=inline
- context: contrib/${{ env.REPONAME }}image/${{ matrix.source }}
- file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile
- platforms: ${{ env.PLATFORMS }}
- push: true
- tags: ${{ steps.containers_reg.outputs.fqin }}
- labels: |
- ${{ env.LABELS }}
diff --git a/contrib/helloimage/Containerfile b/contrib/hello/Containerfile
index 0cbf6d9a0..0cbf6d9a0 100644
--- a/contrib/helloimage/Containerfile
+++ b/contrib/hello/Containerfile
diff --git a/contrib/helloimage/README.md b/contrib/hello/README.md
index 528466f7b..528466f7b 100644
--- a/contrib/helloimage/README.md
+++ b/contrib/hello/README.md
diff --git a/contrib/helloimage/podman_hello_world.c b/contrib/hello/podman_hello_world.c
index ee574130d..ee574130d 100644
--- a/contrib/helloimage/podman_hello_world.c
+++ b/contrib/hello/podman_hello_world.c
diff --git a/contrib/systemd/system/podman.service.in b/contrib/systemd/system/podman.service.in
index 9a7e04fd4..c1a5952b5 100644
--- a/contrib/systemd/system/podman.service.in
+++ b/contrib/systemd/system/podman.service.in
@@ -6,6 +6,7 @@ Documentation=man:podman-system-service(1)
StartLimitIntervalSec=0
[Service]
+Delegate=true
Type=exec
KillMode=process
Environment=LOGGING="--log-level=info"
diff --git a/libpod/define/version.go b/libpod/define/version.go
index 039b0ff27..2c17e6e92 100644
--- a/libpod/define/version.go
+++ b/libpod/define/version.go
@@ -27,6 +27,7 @@ type Version struct {
BuiltTime string
Built int64
OsArch string
+ Os string
}
// GetVersion returns a VersionOutput struct for API and podman
@@ -49,5 +50,6 @@ func GetVersion() (Version, error) {
BuiltTime: time.Unix(buildTime, 0).Format(time.ANSIC),
Built: buildTime,
OsArch: runtime.GOOS + "/" + runtime.GOARCH,
+ Os: runtime.GOOS,
}, nil
}
diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go
index db36ac75d..71e29f18f 100644
--- a/libpod/networking_linux.go
+++ b/libpod/networking_linux.go
@@ -498,10 +498,13 @@ func (r *Runtime) GetRootlessNetNs(new bool) (*RootlessNetNS, error) {
return nil, err
}
- // move to systemd scope to prevent systemd from killing it
- err = utils.MoveRootlessNetnsSlirpProcessToUserSlice(cmd.Process.Pid)
- if err != nil {
- logrus.Errorf("failed to move the rootless netns slirp4netns process to the systemd user.slice: %v", err)
+ if utils.RunsOnSystemd() {
+ // move to systemd scope to prevent systemd from killing it
+ err = utils.MoveRootlessNetnsSlirpProcessToUserSlice(cmd.Process.Pid)
+ if err != nil {
+ // only log this, it is not fatal but can lead to issues when running podman inside systemd units
+ logrus.Errorf("failed to move the rootless netns slirp4netns process to the systemd user.slice: %v", err)
+ }
}
// build a new resolv.conf file which uses the slirp4netns dns server address
diff --git a/libpod/networking_slirp4netns.go b/libpod/networking_slirp4netns.go
index 3f6c4bef2..3f2842d4c 100644
--- a/libpod/networking_slirp4netns.go
+++ b/libpod/networking_slirp4netns.go
@@ -614,60 +614,73 @@ func (r *Runtime) setupRootlessPortMappingViaSlirp(ctr *Container, cmd *exec.Cmd
// for each port we want to add we need to open a connection to the slirp4netns control socket
// and send the add_hostfwd command.
- for _, i := range ctr.convertPortMappings() {
- conn, err := net.Dial("unix", apiSocket)
- if err != nil {
- return errors.Wrapf(err, "cannot open connection to %s", apiSocket)
- }
- defer func() {
- if err := conn.Close(); err != nil {
- logrus.Errorf("Unable to close connection: %q", err)
+ for _, port := range ctr.convertPortMappings() {
+ protocols := strings.Split(port.Protocol, ",")
+ for _, protocol := range protocols {
+ hostIP := port.HostIP
+ if hostIP == "" {
+ hostIP = "0.0.0.0"
+ }
+ for i := uint16(0); i < port.Range; i++ {
+ if err := openSlirp4netnsPort(apiSocket, protocol, hostIP, port.HostPort+i, port.ContainerPort+i); err != nil {
+ return err
+ }
}
- }()
- hostIP := i.HostIP
- if hostIP == "" {
- hostIP = "0.0.0.0"
- }
- apiCmd := slirp4netnsCmd{
- Execute: "add_hostfwd",
- Args: slirp4netnsCmdArg{
- Proto: i.Protocol,
- HostAddr: hostIP,
- HostPort: i.HostPort,
- GuestPort: i.ContainerPort,
- },
- }
- // create the JSON payload and send it. Mark the end of request shutting down writes
- // to the socket, as requested by slirp4netns.
- data, err := json.Marshal(&apiCmd)
- if err != nil {
- return errors.Wrapf(err, "cannot marshal JSON for slirp4netns")
- }
- if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", data))); err != nil {
- return errors.Wrapf(err, "cannot write to control socket %s", apiSocket)
- }
- if err := conn.(*net.UnixConn).CloseWrite(); err != nil {
- return errors.Wrapf(err, "cannot shutdown the socket %s", apiSocket)
- }
- buf := make([]byte, 2048)
- readLength, err := conn.Read(buf)
- if err != nil {
- return errors.Wrapf(err, "cannot read from control socket %s", apiSocket)
- }
- // if there is no 'error' key in the received JSON data, then the operation was
- // successful.
- var y map[string]interface{}
- if err := json.Unmarshal(buf[0:readLength], &y); err != nil {
- return errors.Wrapf(err, "error parsing error status from slirp4netns")
- }
- if e, found := y["error"]; found {
- return errors.Errorf("from slirp4netns while setting up port redirection: %v", e)
}
}
logrus.Debug("slirp4netns port-forwarding setup via add_hostfwd is ready")
return nil
}
+// openSlirp4netnsPort sends the slirp4netns pai quey to the given socket
+func openSlirp4netnsPort(apiSocket, proto, hostip string, hostport, guestport uint16) error {
+ conn, err := net.Dial("unix", apiSocket)
+ if err != nil {
+ return errors.Wrapf(err, "cannot open connection to %s", apiSocket)
+ }
+ defer func() {
+ if err := conn.Close(); err != nil {
+ logrus.Errorf("Unable to close slirp4netns connection: %q", err)
+ }
+ }()
+ apiCmd := slirp4netnsCmd{
+ Execute: "add_hostfwd",
+ Args: slirp4netnsCmdArg{
+ Proto: proto,
+ HostAddr: hostip,
+ HostPort: hostport,
+ GuestPort: guestport,
+ },
+ }
+ // create the JSON payload and send it. Mark the end of request shutting down writes
+ // to the socket, as requested by slirp4netns.
+ data, err := json.Marshal(&apiCmd)
+ if err != nil {
+ return errors.Wrapf(err, "cannot marshal JSON for slirp4netns")
+ }
+ if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", data))); err != nil {
+ return errors.Wrapf(err, "cannot write to control socket %s", apiSocket)
+ }
+ if err := conn.(*net.UnixConn).CloseWrite(); err != nil {
+ return errors.Wrapf(err, "cannot shutdown the socket %s", apiSocket)
+ }
+ buf := make([]byte, 2048)
+ readLength, err := conn.Read(buf)
+ if err != nil {
+ return errors.Wrapf(err, "cannot read from control socket %s", apiSocket)
+ }
+ // if there is no 'error' key in the received JSON data, then the operation was
+ // successful.
+ var y map[string]interface{}
+ if err := json.Unmarshal(buf[0:readLength], &y); err != nil {
+ return errors.Wrapf(err, "error parsing error status from slirp4netns")
+ }
+ if e, found := y["error"]; found {
+ return errors.Errorf("from slirp4netns while setting up port redirection: %v", e)
+ }
+ return nil
+}
+
func getRootlessPortChildIP(c *Container, netStatus map[string]types.StatusBlock) string {
if c.config.NetMode.IsSlirp4netns() {
slirp4netnsIP, err := GetSlirp4netnsIP(c.slirp4netnsSubnet)
diff --git a/pkg/bindings/system/system.go b/pkg/bindings/system/system.go
index 1eb90ab54..5ef78e444 100644
--- a/pkg/bindings/system/system.go
+++ b/pkg/bindings/system/system.go
@@ -120,6 +120,7 @@ func Version(ctx context.Context, options *VersionOptions) (*entities.SystemVers
BuiltTime: time.Unix(b.Unix(), 0).Format(time.ANSIC),
Built: b.Unix(),
OsArch: fmt.Sprintf("%s/%s", component.Os, component.Arch),
+ Os: component.Os,
}
for _, c := range component.Components {
diff --git a/pkg/machine/qemu/config.go b/pkg/machine/qemu/config.go
index 05a1d74d3..3d1032fba 100644
--- a/pkg/machine/qemu/config.go
+++ b/pkg/machine/qemu/config.go
@@ -5,6 +5,7 @@ package qemu
import (
"errors"
+ "io/ioutil"
"os"
"time"
@@ -59,6 +60,8 @@ type MachineVMV1 struct {
}
type MachineVM struct {
+ // ConfigPath is the path to the configuration file
+ ConfigPath MachineFile
// The command line representation of the qemu command
CmdLine []string
// HostUser contains info about host user
@@ -83,11 +86,11 @@ type MachineVM struct {
// ImageConfig describes the bootable image for the VM
type ImageConfig struct {
- IgnitionFilePath string
+ IgnitionFilePath MachineFile
// ImageStream is the update stream for the image
ImageStream string
// ImagePath is the fq path to
- ImagePath string
+ ImagePath MachineFile
}
// HostUser describes the host user
@@ -171,11 +174,19 @@ func (m *MachineFile) GetPath() string {
// the actual path
func (m *MachineFile) Delete() error {
if m.Symlink != nil {
- if err := os.Remove(*m.Symlink); err != nil {
+ if err := os.Remove(*m.Symlink); err != nil && !errors.Is(err, os.ErrNotExist) {
logrus.Errorf("unable to remove symlink %q", *m.Symlink)
}
}
- return os.Remove(m.Path)
+ if err := os.Remove(m.Path); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ return nil
+}
+
+// Read the contents of a given file and return in []bytes
+func (m *MachineFile) Read() ([]byte, error) {
+ return ioutil.ReadFile(m.GetPath())
}
// NewMachineFile is a constructor for MachineFile
diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go
index 07155bbcf..65980129d 100644
--- a/pkg/machine/qemu/machine.go
+++ b/pkg/machine/qemu/machine.go
@@ -71,10 +71,17 @@ func (p *Provider) NewMachine(opts machine.InitOptions) (machine.VM, error) {
if len(opts.Name) > 0 {
vm.Name = opts.Name
}
- ignitionFile := filepath.Join(vmConfigDir, vm.Name+".ign")
- vm.IgnitionFilePath = ignitionFile
+ ignitionFile, err := NewMachineFile(filepath.Join(vmConfigDir, vm.Name+".ign"), nil)
+ if err != nil {
+ return nil, err
+ }
+ vm.IgnitionFilePath = *ignitionFile
- vm.ImagePath = opts.ImagePath
+ imagePath, err := NewMachineFile(opts.ImagePath, nil)
+ if err != nil {
+ return nil, err
+ }
+ vm.ImagePath = *imagePath
vm.RemoteUsername = opts.Username
// Add a random port for ssh
@@ -104,7 +111,7 @@ func (p *Provider) NewMachine(opts machine.InitOptions) (machine.VM, error) {
// Add cpus
cmd = append(cmd, []string{"-smp", strconv.Itoa(int(vm.CPUs))}...)
// Add ignition file
- cmd = append(cmd, []string{"-fw_cfg", "name=opt/com.coreos/config,file=" + vm.IgnitionFilePath}...)
+ cmd = append(cmd, []string{"-fw_cfg", "name=opt/com.coreos/config,file=" + vm.IgnitionFilePath.GetPath()}...)
// Add qmp socket
monitor, err := NewQMPMonitor("unix", vm.Name, defaultQMPTimeout)
if err != nil {
@@ -117,17 +124,19 @@ func (p *Provider) NewMachine(opts machine.InitOptions) (machine.VM, error) {
// Right now the mac address is hardcoded so that the host networking gives it a specific IP address. This is
// why we can only run one vm at a time right now
cmd = append(cmd, []string{"-netdev", "socket,id=vlan,fd=3", "-device", "virtio-net-pci,netdev=vlan,mac=5a:94:ef:e4:0c:ee"}...)
- socketPath, err := getRuntimeDir()
- if err != nil {
+ if err := vm.setReadySocket(); err != nil {
return nil, err
}
- virtualSocketPath := filepath.Join(socketPath, "podman", vm.Name+"_ready.sock")
+
// Add serial port for readiness
cmd = append(cmd, []string{
"-device", "virtio-serial",
- "-chardev", "socket,path=" + virtualSocketPath + ",server=on,wait=off,id=" + vm.Name + "_ready",
+ "-chardev", "socket,path=" + vm.getReadySocket() + ",server=on,wait=off,id=" + vm.Name + "_ready",
"-device", "virtserialport,chardev=" + vm.Name + "_ready" + ",name=org.fedoraproject.port.0"}...)
vm.CmdLine = cmd
+ if err := vm.setQEMUAndPIDSocket(); err != nil {
+ return nil, err
+ }
return vm, nil
}
@@ -165,12 +174,26 @@ func migrateVM(configPath string, config []byte, vm *MachineVM) error {
vm.ResourceConfig = ResourceConfig{}
vm.SSHConfig = SSHConfig{}
+ ignitionFilePath, err := NewMachineFile(old.IgnitionFilePath, nil)
+ if err != nil {
+ return err
+ }
+ imagePath, err := NewMachineFile(old.ImagePath, nil)
+ if err != nil {
+ return err
+ }
+
+ // setReadySocket will stick the entry into the new struct
+ if err := vm.setReadySocket(); err != nil {
+ return err
+ }
+
vm.CPUs = old.CPUs
vm.CmdLine = old.CmdLine
vm.DiskSize = old.DiskSize
vm.IdentityPath = old.IdentityPath
- vm.IgnitionFilePath = old.IgnitionFilePath
- vm.ImagePath = old.ImagePath
+ vm.IgnitionFilePath = *ignitionFilePath
+ vm.ImagePath = *imagePath
vm.ImageStream = old.ImageStream
vm.Memory = old.Memory
vm.Mounts = old.Mounts
@@ -200,31 +223,15 @@ func migrateVM(configPath string, config []byte, vm *MachineVM) error {
return os.Remove(configPath + ".orig")
}
-// LoadByName reads a json file that describes a known qemu vm
+// LoadVMByName reads a json file that describes a known qemu vm
// and returns a vm instance
func (p *Provider) LoadVMByName(name string) (machine.VM, error) {
vm := &MachineVM{Name: name}
vm.HostUser = HostUser{UID: -1} // posix reserves -1, so use it to signify undefined
- vmConfigDir, err := machine.GetConfDir(vmtype)
- if err != nil {
+ if err := vm.update(); err != nil {
return nil, err
}
- path := filepath.Join(vmConfigDir, name+".json")
- b, err := ioutil.ReadFile(path)
- if os.IsNotExist(err) {
- return nil, errors.Wrap(machine.ErrNoSuchVM, name)
- }
- if err != nil {
- return nil, err
- }
- err = json.Unmarshal(b, vm)
- if err != nil {
- migrateErr := migrateVM(path, b, vm)
- if migrateErr != nil {
- return nil, migrateErr
- }
- err = migrateErr
- }
+
// It is here for providing the ability to propagate
// proxy settings (e.g. HTTP_PROXY and others) on a start
// and avoid a need of re-creating/re-initiating a VM
@@ -239,7 +246,7 @@ func (p *Provider) LoadVMByName(name string) (machine.VM, error) {
}
logrus.Debug(vm.CmdLine)
- return vm, err
+ return vm, nil
}
// Init writes the json configuration file to the filesystem for
@@ -261,7 +268,11 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
if err != nil {
return false, err
}
- v.ImagePath = dd.Get().LocalUncompressedFile
+ uncompressedFile, err := NewMachineFile(dd.Get().LocalUncompressedFile, nil)
+ if err != nil {
+ return false, err
+ }
+ v.ImagePath = *uncompressedFile
if err := machine.DownloadImage(dd); err != nil {
return false, err
}
@@ -273,14 +284,17 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
if err != nil {
return false, err
}
- v.ImagePath = g.Get().LocalUncompressedFile
+ imagePath, err := NewMachineFile(g.Get().LocalUncompressedFile, nil)
+ if err != nil {
+ return false, err
+ }
+ v.ImagePath = *imagePath
if err := machine.DownloadImage(g); err != nil {
return false, err
}
}
// Add arch specific options including image location
v.CmdLine = append(v.CmdLine, v.addArchOptions()...)
-
var volumeType string
switch opts.VolumeDriver {
case "virtfs":
@@ -330,7 +344,7 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
v.UID = os.Getuid()
// Add location of bootable image
- v.CmdLine = append(v.CmdLine, "-drive", "if=virtio,file="+v.ImagePath)
+ v.CmdLine = append(v.CmdLine, "-drive", "if=virtio,file="+v.getImageFile())
// This kind of stinks but no other way around this r/n
if len(opts.IgnitionPath) < 1 {
uri := machine.SSHRemoteConnection.MakeSSHURL("localhost", fmt.Sprintf("/run/user/%d/podman/podman.sock", v.UID), strconv.Itoa(v.Port), v.RemoteUsername)
@@ -373,7 +387,7 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
return false, err
}
- originalDiskSize, err := getDiskSize(v.ImagePath)
+ originalDiskSize, err := getDiskSize(v.getImageFile())
if err != nil {
return false, err
}
@@ -390,7 +404,7 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
if err != nil {
return false, err
}
- resize := exec.Command(resizePath, []string{"resize", v.ImagePath, strconv.Itoa(int(opts.DiskSize)) + "G"}...)
+ resize := exec.Command(resizePath, []string{"resize", v.getImageFile(), strconv.Itoa(int(opts.DiskSize)) + "G"}...)
resize.Stdout = os.Stdout
resize.Stderr = os.Stderr
if err := resize.Run(); err != nil {
@@ -404,7 +418,7 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
if err != nil {
return false, err
}
- return false, ioutil.WriteFile(v.IgnitionFilePath, inputIgnition, 0644)
+ return false, ioutil.WriteFile(v.getIgnitionFile(), inputIgnition, 0644)
}
// Write the ignition file
ign := machine.DynamicIgnition{
@@ -412,14 +426,14 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) {
Key: key,
VMName: v.Name,
TimeZone: opts.TimeZone,
- WritePath: v.IgnitionFilePath,
+ WritePath: v.getIgnitionFile(),
UID: v.UID,
}
err = machine.NewIgnitionFile(ign)
return err == nil, err
}
-func (v *MachineVM) Set(name string, opts machine.SetOptions) error {
+func (v *MachineVM) Set(_ string, opts machine.SetOptions) error {
if v.Rootful == opts.Rootful {
return nil
}
@@ -473,17 +487,16 @@ func (v *MachineVM) Start(name string, _ machine.StartOptions) error {
return err
}
}
- qemuSocketPath, _, err := v.getSocketandPid()
- if err != nil {
+ if err := v.setQEMUAndPIDSocket(); err != nil {
return err
}
// If the qemusocketpath exists and the vm is off/down, we should rm
// it before the dial as to avoid a segv
- if err := os.Remove(qemuSocketPath); err != nil && !errors.Is(err, os.ErrNotExist) {
- logrus.Warn(err)
+ if err := v.QMPMonitor.Address.Delete(); err != nil {
+ return err
}
for i := 0; i < 6; i++ {
- qemuSocketConn, err = net.Dial("unix", qemuSocketPath)
+ qemuSocketConn, err = net.Dial("unix", v.QMPMonitor.Address.GetPath())
if err == nil {
break
}
@@ -650,7 +663,7 @@ func (v *MachineVM) checkStatus(monitor *qmp.SocketMonitor) (machine.QemuMachine
}
// Stop uses the qmp monitor to call a system_powerdown
-func (v *MachineVM) Stop(name string, _ machine.StopOptions) error {
+func (v *MachineVM) Stop(_ string, _ machine.StopOptions) error {
var disconnected bool
// check if the qmp socket is there. if not, qemu instance is gone
if _, err := os.Stat(v.QMPMonitor.Address.GetPath()); os.IsNotExist(err) {
@@ -687,14 +700,10 @@ func (v *MachineVM) Stop(name string, _ machine.StopOptions) error {
return err
}
- qemuSocketFile, pidFile, err := v.getSocketandPid()
- if err != nil {
- return err
- }
- if _, err := os.Stat(pidFile); os.IsNotExist(err) {
+ if _, err := os.Stat(v.getPidFile()); os.IsNotExist(err) {
return nil
}
- pidString, err := ioutil.ReadFile(pidFile)
+ pidString, err := v.PidFilePath.Read()
if err != nil {
return err
}
@@ -712,11 +721,11 @@ func (v *MachineVM) Stop(name string, _ machine.StopOptions) error {
return err
}
// Remove the pidfile
- if err := os.Remove(pidFile); err != nil && !errors.Is(err, os.ErrNotExist) {
- logrus.Warn(err)
+ if err := v.PidFilePath.Delete(); err != nil {
+ return err
}
// Remove socket
- if err := os.Remove(qemuSocketFile); err != nil {
+ if err := v.QMPMonitor.Address.Delete(); err != nil {
return err
}
@@ -774,7 +783,7 @@ func NewQMPMonitor(network, name string, timeout time.Duration) (Monitor, error)
}
// Remove deletes all the files associated with a machine including ssh keys, the image itself
-func (v *MachineVM) Remove(name string, opts machine.RemoveOptions) (string, func() error, error) {
+func (v *MachineVM) Remove(_ string, opts machine.RemoveOptions) (string, func() error, error) {
var (
files []string
)
@@ -793,10 +802,10 @@ func (v *MachineVM) Remove(name string, opts machine.RemoveOptions) (string, fun
files = append(files, v.IdentityPath, v.IdentityPath+".pub")
}
if !opts.SaveIgnition {
- files = append(files, v.IgnitionFilePath)
+ files = append(files, v.getIgnitionFile())
}
if !opts.SaveImage {
- files = append(files, v.ImagePath)
+ files = append(files, v.getImageFile())
}
socketPath, err := v.getForwardSocketPath()
if err != nil {
@@ -822,19 +831,15 @@ func (v *MachineVM) Remove(name string, opts machine.RemoveOptions) (string, fun
confirmationMessage += msg + "\n"
}
- // Get path to socket and pidFile before we do any cleanups
- qemuSocketFile, pidFile, errSocketFile := v.getSocketandPid()
//silently try to delete socket and pid file
//remove socket and pid file if any: warn at low priority if things fail
- if errSocketFile == nil {
- // Remove the pidfile
- if err := os.Remove(pidFile); err != nil && !errors.Is(err, os.ErrNotExist) {
- logrus.Debugf("Error while removing pidfile: %v", err)
- }
- // Remove socket
- if err := os.Remove(qemuSocketFile); err != nil && !errors.Is(err, os.ErrNotExist) {
- logrus.Debugf("Error while removing podman-machine-socket: %v", err)
- }
+ // Remove the pidfile
+ if err := v.PidFilePath.Delete(); err != nil {
+ logrus.Debugf("Error while removing pidfile: %v", err)
+ }
+ // Remove socket
+ if err := v.QMPMonitor.Address.Delete(); err != nil {
+ logrus.Debugf("Error while removing podman-machine-socket: %v", err)
}
confirmationMessage += "\n"
@@ -890,7 +895,7 @@ func (v *MachineVM) isListening() bool {
// SSH opens an interactive SSH session to the vm specified.
// Added ssh function to VM interface: pkg/machine/config/go : line 58
-func (v *MachineVM) SSH(name string, opts machine.SSHOptions) error {
+func (v *MachineVM) SSH(_ string, opts machine.SSHOptions) error {
running, err := v.isRunning()
if err != nil {
return err
@@ -966,10 +971,10 @@ func getDiskSize(path string) (uint64, error) {
// List lists all vm's that use qemu virtualization
func (p *Provider) List(_ machine.ListOptions) ([]*machine.ListResponse, error) {
- return GetVMInfos()
+ return getVMInfos()
}
-func GetVMInfos() ([]*machine.ListResponse, error) {
+func getVMInfos() ([]*machine.ListResponse, error) {
vmConfigDir, err := machine.GetConfDir(vmtype)
if err != nil {
return nil, err
@@ -1011,7 +1016,7 @@ func GetVMInfos() ([]*machine.ListResponse, error) {
}
listEntry.CreatedAt = fi.ModTime()
- fi, err = os.Stat(vm.ImagePath)
+ fi, err = os.Stat(vm.getImageFile())
if err != nil {
return err
}
@@ -1034,7 +1039,7 @@ func GetVMInfos() ([]*machine.ListResponse, error) {
}
func (p *Provider) IsValidVMName(name string) (bool, error) {
- infos, err := GetVMInfos()
+ infos, err := getVMInfos()
if err != nil {
return false, err
}
@@ -1049,7 +1054,7 @@ func (p *Provider) IsValidVMName(name string) (bool, error) {
// CheckExclusiveActiveVM checks if there is a VM already running
// that does not allow other VMs to be running
func (p *Provider) CheckExclusiveActiveVM() (bool, string, error) {
- vms, err := GetVMInfos()
+ vms, err := getVMInfos()
if err != nil {
return false, "", errors.Wrap(err, "error checking VM active")
}
@@ -1073,16 +1078,12 @@ func (v *MachineVM) startHostNetworking() (string, apiForwardingState, error) {
return "", noForwarding, err
}
- qemuSocket, pidFile, err := v.getSocketandPid()
- if err != nil {
- return "", noForwarding, err
- }
attr := new(os.ProcAttr)
// Pass on stdin, stdout, stderr
files := []*os.File{os.Stdin, os.Stdout, os.Stderr}
attr.Files = files
cmd := []string{binary}
- cmd = append(cmd, []string{"-listen-qemu", fmt.Sprintf("unix://%s", qemuSocket), "-pid-file", pidFile}...)
+ cmd = append(cmd, []string{"-listen-qemu", fmt.Sprintf("unix://%s", v.getQMPMonitorSocket()), "-pid-file", v.getPidFile()}...)
// Add the ssh port
cmd = append(cmd, []string{"-ssh-port", fmt.Sprintf("%d", v.Port)}...)
@@ -1171,6 +1172,52 @@ func (v *MachineVM) getForwardSocketPath() (string, error) {
return filepath.Join(path, "podman.sock"), nil
}
+func (v *MachineVM) setConfigPath() error {
+ vmConfigDir, err := machine.GetConfDir(vmtype)
+ if err != nil {
+ return err
+ }
+
+ configPath, err := NewMachineFile(filepath.Join(vmConfigDir, v.Name)+".json", nil)
+ if err != nil {
+ return err
+ }
+ v.ConfigPath = *configPath
+ return nil
+}
+
+func (v *MachineVM) setReadySocket() error {
+ rtPath, err := getRuntimeDir()
+ if err != nil {
+ return err
+ }
+ virtualSocketPath, err := NewMachineFile(filepath.Join(rtPath, "podman", v.Name+"_ready.sock"), nil)
+ if err != nil {
+ return err
+ }
+ v.ReadySocket = *virtualSocketPath
+ return nil
+}
+
+func (v *MachineVM) setQEMUAndPIDSocket() error {
+ rtPath, err := getRuntimeDir()
+ if err != nil {
+ return err
+ }
+ if !rootless.IsRootless() {
+ rtPath = "/run"
+ }
+ socketDir := filepath.Join(rtPath, "podman")
+ pidFilePath, err := NewMachineFile(filepath.Join(socketDir, fmt.Sprintf("%s.pid", v.Name)), nil)
+ if err != nil {
+ return err
+ }
+ v.PidFilePath = *pidFilePath
+ return nil
+}
+
+// Deprecated: getSocketandPid is being replace by setQEMUAndPIDSocket and
+// machinefiles.
func (v *MachineVM) getSocketandPid() (string, string, error) {
rtPath, err := getRuntimeDir()
if err != nil {
@@ -1287,23 +1334,73 @@ func (v *MachineVM) waitAPIAndPrintInfo(forwardState apiForwardingState, forward
}
}
-func (v *MachineVM) writeConfig() error {
- // GetConfDir creates the directory so no need to check for
- // its existence
- vmConfigDir, err := machine.GetConfDir(vmtype)
+// update returns the content of the VM's
+// configuration file in json
+func (v *MachineVM) update() error {
+ if err := v.setConfigPath(); err != nil {
+ return err
+ }
+ b, err := v.ConfigPath.Read()
if err != nil {
+ if os.IsNotExist(err) {
+ return errors.Wrap(machine.ErrNoSuchVM, v.Name)
+ }
return err
}
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(b, v)
+ if err != nil {
+ err = migrateVM(v.ConfigPath.GetPath(), b, v)
+ if err != nil {
+ return err
+ }
+ }
+ return err
+}
- jsonFile := filepath.Join(vmConfigDir, v.Name) + ".json"
+func (v *MachineVM) writeConfig() error {
+ // Set the path of the configfile before writing to make
+ // life easier down the line
+ if err := v.setConfigPath(); err != nil {
+ return err
+ }
// Write the JSON file
b, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
- if err := ioutil.WriteFile(jsonFile, b, 0644); err != nil {
+ if err := ioutil.WriteFile(v.ConfigPath.GetPath(), b, 0644); err != nil {
return err
}
-
return nil
}
+
+// getPidFile gets the file where the machine pid is stored
+func (v *MachineVM) getPidFile() string {
+ return v.PidFilePath.GetPath()
+}
+
+// getQMPMonitorSocket gets the socket used by qemu to interact
+// with the instance
+func (v *MachineVM) getQMPMonitorSocket() string {
+ return v.QMPMonitor.Address.GetPath()
+}
+
+// getReadySocket returns the socket used to communicate
+// with the machinevm and report when it is booted
+func (v *MachineVM) getReadySocket() string {
+ return v.ReadySocket.GetPath()
+}
+
+// getImageFile returns the path to the image used
+// to boot the VM
+func (v *MachineVM) getImageFile() string {
+ return v.ImagePath.GetPath()
+}
+
+// getIgnitionFile returns the path to the ignition file
+func (v *MachineVM) getIgnitionFile() string {
+ return v.IgnitionFilePath.GetPath()
+}
diff --git a/pkg/machine/qemu/options_darwin_arm64.go b/pkg/machine/qemu/options_darwin_arm64.go
index 5b6cdc86d..4c954af00 100644
--- a/pkg/machine/qemu/options_darwin_arm64.go
+++ b/pkg/machine/qemu/options_darwin_arm64.go
@@ -11,7 +11,7 @@ var (
)
func (v *MachineVM) addArchOptions() []string {
- ovmfDir := getOvmfDir(v.ImagePath, v.Name)
+ ovmfDir := getOvmfDir(v.ImagePath.GetPath(), v.Name)
opts := []string{
"-accel", "hvf",
"-accel", "tcg",
@@ -23,13 +23,13 @@ func (v *MachineVM) addArchOptions() []string {
}
func (v *MachineVM) prepare() error {
- ovmfDir := getOvmfDir(v.ImagePath, v.Name)
+ ovmfDir := getOvmfDir(v.ImagePath.GetPath(), v.Name)
cmd := []string{"/bin/dd", "if=/dev/zero", "conv=sync", "bs=1m", "count=64", "of=" + ovmfDir}
return exec.Command(cmd[0], cmd[1:]...).Run()
}
func (v *MachineVM) archRemovalFiles() []string {
- ovmDir := getOvmfDir(v.ImagePath, v.Name)
+ ovmDir := getOvmfDir(v.ImagePath.GetPath(), v.Name)
return []string{ovmDir}
}
diff --git a/test/e2e/version_test.go b/test/e2e/version_test.go
index 6f93beff6..a30db80eb 100644
--- a/test/e2e/version_test.go
+++ b/test/e2e/version_test.go
@@ -87,10 +87,18 @@ var _ = Describe("Podman version", func() {
session.WaitWithDefaultTimeout()
Expect(session).Should(Exit(0))
+ session = podmanTest.Podman([]string{"version", "--format", "{{ .Client.Os }}"})
+ session.WaitWithDefaultTimeout()
+ Expect(session).Should(Exit(0))
+
session = podmanTest.Podman([]string{"version", "--format", "{{ .Server.Version }}"})
session.WaitWithDefaultTimeout()
Expect(session).Should(Exit(0))
+ session = podmanTest.Podman([]string{"version", "--format", "{{ .Server.Os }}"})
+ session.WaitWithDefaultTimeout()
+ Expect(session).Should(Exit(0))
+
session = podmanTest.Podman([]string{"version", "--format", "{{ .Version }}"})
session.WaitWithDefaultTimeout()
Expect(session).Should(Exit(0))
diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats
index a95561635..78ad3fe04 100644
--- a/test/system/500-networking.bats
+++ b/test/system/500-networking.bats
@@ -632,4 +632,25 @@ EOF
is "$output" ".*nameserver $subnet.1.*" "integrated dns nameserver is set"
}
+@test "podman run port forward range" {
+ for netmode in bridge slirp4netns:port_handler=slirp4netns slirp4netns:port_handler=rootlesskit; do
+ local port=$(random_free_port)
+ local end_port=$(( $port + 2 ))
+ local range="$port-$end_port:$port-$end_port"
+ local random=$(random_string)
+
+ run_podman run --network $netmode -p "$range" -d $IMAGE sleep inf
+ cid="$output"
+ for port in $(seq $port $end_port); do
+ run_podman exec -d $cid nc -l -p $port -e /bin/cat
+ # -w 1 adds a 1 second timeout, for some reason ubuntus ncat doesn't close the connection on EOF,
+ # other options to change this are not portable across distros but -w seems to work
+ run nc -w 1 127.0.0.1 $port <<<$random
+ is "$output" "$random" "ncat got data back (netmode=$netmode port=$port)"
+ done
+
+ run_podman rm -f -t0 $cid
+ done
+}
+
# vim: filetype=sh
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-11-08 03:44:01 +0000