diff options
| -rw-r--r-- | pkg/machine/fcos.go | 57 | ||||
| -rw-r--r-- | troubleshooting.md | 212 | ||||
| -rw-r--r-- | vendor/github.com/coreos/stream-metadata-go/release/release.go | 112 | ||||
| -rw-r--r-- | vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go | 14 | ||||
| -rw-r--r-- | vendor/github.com/coreos/stream-metadata-go/release/translate.go | 196 | ||||
| -rw-r--r-- | vendor/modules.txt | 2 |
6 files changed, 591 insertions, 2 deletions
diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go index 60ab471ee..4d3e2edf4 100644 --- a/pkg/machine/fcos.go +++ b/pkg/machine/fcos.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/coreos/stream-metadata-go/fedoracoreos" + "github.com/coreos/stream-metadata-go/release" "github.com/coreos/stream-metadata-go/stream" "github.com/pkg/errors" @@ -28,6 +29,14 @@ var ( Format string = "qcow2.xz" ) +const ( + // Used for testing the latest podman in fcos + // special builds + podmanTesting = "podman-testing" + PodmanTestingHost = "fedorapeople.org" + PodmanTestingURL = "groups/podman/testing" +) + type FcosDownload struct { Download } @@ -111,14 +120,39 @@ func getFcosArch() string { return arch } +// getStreamURL is a wrapper for the fcos.GetStream URL +// so that we can inject a special stream and url for +// testing podman before it merges into fcos builds +func getStreamURL(streamType string) url2.URL { + // For the podmanTesting stream type, we point to + // a custom url on fedorapeople.org + if streamType == podmanTesting { + return url2.URL{ + Scheme: "https", + Host: PodmanTestingHost, + Path: fmt.Sprintf("%s/%s.json", PodmanTestingURL, "podman4"), + } + } + return fedoracoreos.GetStreamURL(streamType) +} + // This should get Exported and stay put as it will apply to all fcos downloads // getFCOS parses fedoraCoreOS's stream and returns the image download URL and the release version func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { var ( fcosstable stream.Stream + altMeta release.Release streamType string ) + + // This is being hard set to testing. Once podman4 is in the + // fcos trees, we should remove it and re-release at least on + // macs. + imageStream = "podman-testing" + switch imageStream { + case "podman-testing": + streamType = "podman-testing" case "testing", "": streamType = fedoracoreos.StreamTesting case "next": @@ -128,7 +162,7 @@ func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { default: return nil, errors.Errorf("invalid stream %s: valid streams are `testing` and `stable`", imageStream) } - streamurl := fedoracoreos.GetStreamURL(streamType) + streamurl := getStreamURL(streamType) resp, err := http.Get(streamurl.String()) if err != nil { return nil, err @@ -142,6 +176,27 @@ func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { logrus.Error(err) } }() + if imageStream == podmanTesting { + if err := json.Unmarshal(body, &altMeta); err != nil { + return nil, err + } + + arches, ok := altMeta.Architectures[getFcosArch()] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no targetArch in stream") + } + qcow2, ok := arches.Media.Qemu.Artifacts["qcow2.xz"] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qcow2.xz format in stream") + } + disk := qcow2.Disk + + return &fcosDownloadInfo{ + Location: disk.Location, + Sha256Sum: disk.Sha256, + CompressionType: "xz", + }, nil + } if err := json.Unmarshal(body, &fcosstable); err != nil { return nil, err diff --git a/troubleshooting.md b/troubleshooting.md index 8fcce22a7..6f2a96a56 100644 --- a/troubleshooting.md +++ b/troubleshooting.md @@ -916,7 +916,7 @@ After deleting a VM on macOS, the initialization of subsequent VMs fails. After deleting a client VM on macOS via `podman machine stop` && `podman machine rm`, attempting to `podman machine init` a new client VM leads to an error with the 127.0.0.1:7777 port already bound. -### Solution +#### Solution You will need to remove the hanging gv-proxy process bound to the port in question. For example, if the port mentioned in the error message is 127.0.0.1:7777, you can use the command `kill -9 $(lsof -i:7777)` in order to identify and remove the hanging process which prevents you from starting a new VM on that default port. @@ -938,3 +938,213 @@ run Podman from a system service, either using the Podman service, and then using podman -remote to start the container or simply by running something like `systemd-run podman run ...`. In this case the container will only need `CAP_AUDIT_WRITE`. + +### 33) Container creates a file that is not owned by the user's regular UID + +After running a container with rootless Podman, the non-root user sees a numerical UID and GID instead of a username and groupname. + +#### Symptom + +When listing file permissions with `ls -l` on the host in a directory that was passed as `--volume /some/dir` to `podman run`, +the UID and GID are displayed rather than the corresponding username and groupname. The UID and GID numbers displayed are +from the user's subordinate UID and GID ranges on the host system. + +An example + +```Text +$ mkdir dir1 +$ chmod 777 dir1 +$ podman run --rm -v ./dir1:/dir1:Z \ + --user 2003:2003 \ + docker.io/library/ubuntu bash -c "touch /dir1/a; chmod 600 /dir1/a" +$ ls -l dir1/a +-rw-------. 1 102002 102002 0 Jan 19 19:35 dir1/a +$ less dir1/a +less: dir1/a: Permission denied +``` + +#### Solution + +If you want to read or remove such a file, you can do so by entering a user namespace. +Instead of running commands such as `less dir1/a` or `rm dir1/a`, you would need to +prepend the command-line with `podman unshare`, i.e., +`podman unshare less dir1/a` or `podman unshare rm dir1/a`. To be able to use Bash +features, such as variable expansion and globbing, you need to wrap the command with +`bash -c`, e.g. `podman unshare bash -c 'ls $HOME/dir1/a*'`. + +Would it have been possible to run Podman in another way so that your regular +user would have become the owner of the file? Yes, you can use the options +__--uidmap__ and __--gidmap__ to change how UIDs and GIDs are mapped +between the container and the host. Let's try it out. + +In the example above `ls -l` shows the UID 102002 and GID 102002. Set shell variables + +```Text +$ uid_from_ls = 102002 +$ gid_from_ls = 102002 +``` + +Set shell variables to the lowest subordinate UID and GID + +```Text +$ lowest_subuid=$(podman info --format "{{ (index .Host.IDMappings.UIDMap 1).HostID }}") +$ lowest_subgid=$(podman info --format "{{ (index .Host.IDMappings.GIDMap 1).HostID }}") +``` + +Compute the UID and GID inside the container that map to the owner of the created file on the host. + +```Text +$ uid=$(( $uid_from_ls - $lowest_subuid + 1)) +$ gid=$(( $gid_from_ls - $lowest_subgid + 1)) +``` +(In the computation it was assumed that there is only one subuid range and one subgid range) + +```Text +$ echo $uid +2003 +$ echo $gid +2003 +``` + +The computation shows that the UID is _2003_ and the GID is _2003_ inside the container. +This comes as no surprise as this is what was specified before with `--user=2003:2003`, +but the same computation could be used whenever a username is specified +or the __--user__ option is not used. + +Run the container again but now with UIDs and GIDs mapped + +```Text +$ subuidSize=$(( $(podman info --format "{{ range .Host.IDMappings.UIDMap }}+{{.Size }}{{end }}" ) - 1 )) +$ subgidSize=$(( $(podman info --format "{{ range .Host.IDMappings.GIDMap }}+{{.Size }}{{end }}" ) - 1 )) +$ mkdir dir1 +$ chmod 777 dir1 +$ podman run --rm + -v ./dir1:/dir1:Z \ + --user $uid:$gid \ + --uidmap $uid:0:1 \ + --uidmap 0:1:$uid \ + --uidmap $(($uid+1)):$(($uid+1)):$(($subuidSize-$uid)) \ + --gidmap $gid:0:1 \ + --gidmap 0:1:$gid \ + --gidmap $(($gid+1)):$(($gid+1)):$(($subgidSize-$gid)) \ + docker.io/library/ubuntu bash -c "touch /dir1/a; chmod 600 /dir1/a" +$ id -u +tester +$ id -g +tester +$ ls -l dir1/a +-rw-------. 1 tester tester 0 Jan 19 20:31 dir1/a +$ +``` + +In this example the __--user__ option specified a rootless user in the container. +As the rootless user could also have been specified in the container image, e.g., + +```Text +$ podman image inspect --format "user: {{.User}}" IMAGE +user: hpc +$ +``` +the same problem could also occur even without specifying __--user__. + +Another variant of the same problem could occur when using +__--user=root:root__ (the default), but where the root user creates non-root owned files +in some way (e.g by creating them themselves, or switching the effective UID to +a rootless user and then creates files). + +### 34) Passed-in devices or files can't be accessed in rootless container (UID/GID mapping problem) + +As a non-root user you have access rights to devices, files and directories that you +want to pass into a rootless container with `--device=...`, `--volume=...` or `--mount=..`. + +Podman by default maps a non-root user inside a container to one of the user's +subordinate UIDs and subordinate GIDs on the host. When the container user tries to access a +file, a "Permission denied" error could occur because the container user does not have the +permissions of the regular user of the host. + +#### Symptom + +* Any access inside the container is rejected with "Permission denied" +for files, directories or devices passed in to the container +with `--device=..`,`--volume=..` or `--mount=..`, e.g. + +```Text +$ mkdir dir1 +$ chmod 700 dir1 +$ podman run --rm -v ./dir1:/dir1:Z \ + --user 2003:2003 \ + docker.io/library/ubuntu ls /dir1 +ls: cannot open directory '/dir1': Permission denied +``` + +#### Solution + +We follow essentialy the same solution as in the previous +troubleshooting tip: + "_Container creates a file that is not owned by the regular UID_" +but for this problem the container UID and GID can't be as +easily computed by mere addition and subtraction. + +In other words, it might be more challenging to find out the UID and +the GID inside the container that we want to map to the regular +user on the host. + +If the __--user__ option is used together with a numerical UID and GID +to specify a rootless user, we already know the answer. + +If the __--user__ option is used together with a username and groupname, +we could look up the UID and GID in the file _/etc/passwd_ of the container. + +If the container user is not set via __--user__ but instead from the +container image, we could inspect the container image + +```Text +$ podman image inspect --format "user: {{.User}}" IMAGE +user: hpc +$ +``` + +and then look it up in _/etc/passwd_ of the container. + +If the problem occurs in a container that is started to run as root but later +switches to an effictive UID of a rootless user, it might be less +straightforward to find out the UID and the GID. Reading the +_Containerfile_, _Dockerfile_ or the _/etc/passwd_ could give a clue. + +To run the container with the rootless container UID and GID mapped to the +user's regular UID and GID on the host follow these steps: + +Set the _uid_ and _gid_ shell variables in a Bash shell to the UID and GID +of the user that will be running inside the container, e.g. + +```Text +$ uid=2003 +$ gid=2003 +``` + +and run + +```Text +$ mkdir dir1 +$ echo hello > dir1/file.txt +$ chmod 700 dir1/file.txt +$ subuidSize=$(( $(podman info --format "{{ range .Host.IDMappings.UIDMap }}+{{.Size }}{{end }}" ) - 1 )) +$ subgidSize=$(( $(podman info --format "{{ range .Host.IDMappings.GIDMap }}+{{.Size }}{{end }}" ) - 1 )) +$ podman run --rm \ + -v ./dir1:/dir1:Z \ + --user $uid:$gid \ + --uidmap $uid:0:1 \ + --uidmap 0:1:$uid \ + --uidmap $(($uid+1)):$(($uid+1)):$(($subuidSize-$uid)) \ + --gidmap $gid:0:1 \ + --gidmap 0:1:$gid \ + --gidmap $(($gid+1)):$(($gid+1)):$(($subgidSize-$gid)) \ + docker.io/library/alpine cat /dir1/file.txt +hello +$ +``` + +A side-note: Using [__--userns=keep-id__](https://docs.podman.io/en/latest/markdown/podman-run.1.html#userns-mode) +can sometimes be an alternative solution, but it forces the regular +user's host UID to be mapped to the same UID inside the container +so it provides less flexibility than using __--uidmap__ and __--gidmap__. diff --git a/vendor/github.com/coreos/stream-metadata-go/release/release.go b/vendor/github.com/coreos/stream-metadata-go/release/release.go new file mode 100644 index 000000000..84a032703 --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/release.go @@ -0,0 +1,112 @@ +// Package release contains APIs for interacting with a +// particular "release". Avoid this unless you are sure +// you need it. It's expected that CoreOS users interact +// with streams instead. +package release + +import ( + relrhcos "github.com/coreos/stream-metadata-go/release/rhcos" +) + +// Index models the release index: +// https://github.com/coreos/fedora-coreos-tracker/tree/master/metadata/release-index +type Index struct { + Note string `json:"note"` // used to note to users not to consume the release metadata index + Releases []IndexRelease `json:"releases"` + Metadata Metadata `json:"metadata"` + Stream string `json:"stream"` +} + +// IndexRelease is a "release pointer" from a release index +type IndexRelease struct { + Commits []IndexReleaseCommit `json:"commits"` + Version string `json:"version"` + MetadataURL string `json:"metadata"` +} + +// IndexReleaseCommit describes an ostree commit plus architecture +type IndexReleaseCommit struct { + Architecture string `json:"architecture"` + Checksum string `json:"checksum"` +} + +// Release contains details from release.json +type Release struct { + Release string `json:"release"` + Stream string `json:"stream"` + Metadata Metadata `json:"metadata"` + Architectures map[string]Arch `json:"architectures"` +} + +// Metadata is common metadata that contains last-modified +type Metadata struct { + LastModified string `json:"last-modified"` +} + +// Arch release details +type Arch struct { + Commit string `json:"commit"` + Media Media `json:"media"` + RHELCoreOSExtensions *relrhcos.Extensions `json:"rhel-coreos-extensions,omitempty"` +} + +// Media contains release details for various platforms +type Media struct { + Aliyun *PlatformBase `json:"aliyun"` + Aws *PlatformAws `json:"aws"` + Azure *PlatformBase `json:"azure"` + Digitalocean *PlatformBase `json:"digitalocean"` + Exoscale *PlatformBase `json:"exoscale"` + Gcp *PlatformGcp `json:"gcp"` + Ibmcloud *PlatformBase `json:"ibmcloud"` + Metal *PlatformBase `json:"metal"` + Openstack *PlatformBase `json:"openstack"` + Qemu *PlatformBase `json:"qemu"` + Vmware *PlatformBase `json:"vmware"` + Vultr *PlatformBase `json:"vultr"` +} + +// PlatformBase with no cloud images +type PlatformBase struct { + Artifacts map[string]ImageFormat `json:"artifacts"` +} + +// PlatformAws contains AWS image information +type PlatformAws struct { + PlatformBase + Images map[string]CloudImage `json:"images"` +} + +// PlatformGcp GCP image detail +type PlatformGcp struct { + PlatformBase + Image *GcpImage `json:"image"` +} + +// ImageFormat contains all artifacts for a single OS image +type ImageFormat struct { + Disk *Artifact `json:"disk,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Rootfs *Artifact `json:"rootfs,omitempty"` +} + +// Artifact represents one image file, plus its metadata +type Artifact struct { + Location string `json:"location"` + Signature string `json:"signature"` + Sha256 string `json:"sha256"` + UncompressedSha256 string `json:"uncompressed-sha256,omitempty"` +} + +// CloudImage generic image detail +type CloudImage struct { + Image string `json:"image"` +} + +// GcpImage represents a GCP cloud image +type GcpImage struct { + Project string `json:"project,omitempty"` + Family string `json:"family,omitempty"` + Name string `json:"name,omitempty"` +} diff --git a/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go b/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go new file mode 100644 index 000000000..aeae2c8be --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go @@ -0,0 +1,14 @@ +package rhcos + +// Extensions is data specific to Red Hat Enterprise Linux CoreOS +type Extensions struct { + AzureDisk *AzureDisk `json:"azure-disk,omitempty"` +} + +// AzureDisk represents an Azure cloud image. +type AzureDisk struct { + // URL to an image already stored in Azure infrastructure + // that can be copied into an image gallery. Avoid creating VMs directly + // from this URL as that may lead to performance limitations. + URL string `json:"url,omitempty"` +} diff --git a/vendor/github.com/coreos/stream-metadata-go/release/translate.go b/vendor/github.com/coreos/stream-metadata-go/release/translate.go new file mode 100644 index 000000000..518c75eb9 --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/translate.go @@ -0,0 +1,196 @@ +package release + +import ( + "github.com/coreos/stream-metadata-go/stream" + "github.com/coreos/stream-metadata-go/stream/rhcos" +) + +func mapArtifact(ra *Artifact) *stream.Artifact { + if ra == nil { + return nil + } + return &stream.Artifact{ + Location: ra.Location, + Signature: ra.Signature, + Sha256: ra.Sha256, + UncompressedSha256: ra.UncompressedSha256, + } +} + +func mapFormats(m map[string]ImageFormat) map[string]stream.ImageFormat { + r := make(map[string]stream.ImageFormat) + for k, v := range m { + r[k] = stream.ImageFormat{ + Disk: mapArtifact(v.Disk), + Kernel: mapArtifact(v.Kernel), + Initramfs: mapArtifact(v.Initramfs), + Rootfs: mapArtifact(v.Rootfs), + } + } + return r +} + +// Convert a release architecture to a stream architecture +func (releaseArch *Arch) toStreamArch(rel *Release) stream.Arch { + artifacts := make(map[string]stream.PlatformArtifacts) + cloudImages := stream.Images{} + var rhcosExt *rhcos.Extensions + relRHCOSExt := releaseArch.RHELCoreOSExtensions + if relRHCOSExt != nil { + rhcosExt = &rhcos.Extensions{} + } + if releaseArch.Media.Aws != nil { + artifacts["aws"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aws.Artifacts), + } + awsAmis := stream.AwsImage{ + Regions: make(map[string]stream.AwsRegionImage), + } + if releaseArch.Media.Aws.Images != nil { + for region, ami := range releaseArch.Media.Aws.Images { + ri := stream.AwsRegionImage{Release: rel.Release, Image: ami.Image} + awsAmis.Regions[region] = ri + + } + cloudImages.Aws = &awsAmis + } + } + + if releaseArch.Media.Azure != nil { + artifacts["azure"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Azure.Artifacts), + } + + if relRHCOSExt != nil { + az := relRHCOSExt.AzureDisk + if az != nil { + rhcosExt.AzureDisk = &rhcos.AzureDisk{ + Release: rel.Release, + URL: az.URL, + } + } + } + // In the future this is where we'd also add FCOS Marketplace data. + // See https://github.com/coreos/stream-metadata-go/issues/13 + } + + if releaseArch.Media.Aliyun != nil { + artifacts["aliyun"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aliyun.Artifacts), + } + } + + if releaseArch.Media.Exoscale != nil { + artifacts["exoscale"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Exoscale.Artifacts), + } + } + + if releaseArch.Media.Vultr != nil { + artifacts["vultr"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vultr.Artifacts), + } + } + + if releaseArch.Media.Gcp != nil { + artifacts["gcp"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Gcp.Artifacts), + } + + if releaseArch.Media.Gcp.Image != nil { + cloudImages.Gcp = &stream.GcpImage{ + Name: releaseArch.Media.Gcp.Image.Name, + Family: releaseArch.Media.Gcp.Image.Family, + Project: releaseArch.Media.Gcp.Image.Project, + } + } + } + + if releaseArch.Media.Digitalocean != nil { + artifacts["digitalocean"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Digitalocean.Artifacts), + } + + /* We're producing artifacts but they're not yet available + in DigitalOcean as distribution images. + digitalOceanImage := stream.CloudImage{Image: fmt.Sprintf("fedora-coreos-%s", Stream)} + cloudImages.Digitalocean = &digitalOceanImage + */ + } + + if releaseArch.Media.Ibmcloud != nil { + artifacts["ibmcloud"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Ibmcloud.Artifacts), + } + } + + // if releaseArch.Media.Packet != nil { + // packet := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Packet.Artifacts, + // } + // artifacts.Packet = &packet + + // packetImage := StreamCloudImage{Image: fmt.Sprintf("fedora_coreos_%s", rel.Stream)} + // cloudImages.Packet = &packetImage + // } + + if releaseArch.Media.Openstack != nil { + artifacts["openstack"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Openstack.Artifacts), + } + } + + if releaseArch.Media.Qemu != nil { + artifacts["qemu"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Qemu.Artifacts), + } + } + + // if releaseArch.Media.Virtualbox != nil { + // virtualbox := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Virtualbox.Artifacts, + // } + // artifacts.Virtualbox = &virtualbox + // } + + if releaseArch.Media.Vmware != nil { + artifacts["vmware"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vmware.Artifacts), + } + } + + if releaseArch.Media.Metal != nil { + artifacts["metal"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Metal.Artifacts), + } + } + + return stream.Arch{ + Artifacts: artifacts, + Images: cloudImages, + RHELCoreOSExtensions: rhcosExt, + } +} + +// ToStreamArchitectures converts a release to a stream +func (rel *Release) ToStreamArchitectures() map[string]stream.Arch { + streamArch := make(map[string]stream.Arch) + for arch, releaseArch := range rel.Architectures { + streamArch[arch] = releaseArch.toStreamArch(rel) + } + return streamArch +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 9260511f5..f6042a041 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -291,6 +291,8 @@ github.com/coreos/go-systemd/v22/sdjournal ## explicit github.com/coreos/stream-metadata-go/fedoracoreos github.com/coreos/stream-metadata-go/fedoracoreos/internals +github.com/coreos/stream-metadata-go/release +github.com/coreos/stream-metadata-go/release/rhcos github.com/coreos/stream-metadata-go/stream github.com/coreos/stream-metadata-go/stream/rhcos # github.com/cyphar/filepath-securejoin v0.2.3 |