From 17dbce2fb060b4803b2dae4eb6b78fdebea5b61f Mon Sep 17 00:00:00 2001 From: Ashley Cui Date: Fri, 8 Jul 2022 20:10:25 -0400 Subject: Clean up cached machine images When initing machines, we download a machine image, and uncompress and copy the image for the actual vm image. When a user constantly pulls new machines, there may be a buildup of old, unused machine images. This commit cleans ups the unused cached images. Changes: - If the machine is pulled from a URL or from the FCOS releases, we pull them into XDG_DATA_HOME/containers/podman/machine/vmType/cache - Cache cleanups only happen if there is a cache miss, and we need to pull a new image - For Fedora and FCOS, we actually use the cache, so we go through the cache dir and remove any images older than 2 weeks (FCOS's release cycle), on a cache miss. - For generic files pulled from a URL, we don't actually cache, so we delete the pulled file immediately after creating a machine image - For generic files from a local path, the original file will never be cleaned up Note that because we cache in a different dir, this will not clean up old images pulled before this commit. [NO NEW TESTS NEEDED] Signed-off-by: Ashley Cui --- pkg/machine/pull.go | 54 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 8 deletions(-) (limited to 'pkg/machine/pull.go') diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go index 7e6f01bad..08baa7df8 100644 --- a/pkg/machine/pull.go +++ b/pkg/machine/pull.go @@ -5,6 +5,7 @@ package machine import ( "bufio" + "errors" "fmt" "io" "io/ioutil" @@ -39,6 +40,10 @@ func NewGenericDownloader(vmType, vmName, pullPath string) (DistributionDownload if err != nil { return nil, err } + cacheDir, err := GetCacheDir(vmType) + if err != nil { + return nil, err + } dl := Download{} // Is pullpath a file or url? getURL, err := url2.Parse(pullPath) @@ -48,25 +53,23 @@ func NewGenericDownloader(vmType, vmName, pullPath string) (DistributionDownload if len(getURL.Scheme) > 0 { urlSplit := strings.Split(getURL.Path, "/") imageName = urlSplit[len(urlSplit)-1] - dl.LocalUncompressedFile = filepath.Join(dataDir, imageName) dl.URL = getURL - dl.LocalPath = filepath.Join(dataDir, imageName) + dl.LocalPath = filepath.Join(cacheDir, imageName) } else { // Dealing with FilePath imageName = filepath.Base(pullPath) - dl.LocalUncompressedFile = filepath.Join(dataDir, imageName) dl.LocalPath = pullPath } dl.VMName = vmName dl.ImageName = imageName + dl.LocalUncompressedFile = filepath.Join(dataDir, imageName) // The download needs to be pulled into the datadir gd := GenericDownload{Download: dl} - gd.LocalUncompressedFile = gd.getLocalUncompressedName() return gd, nil } -func (d Download) getLocalUncompressedName() string { +func (d Download) getLocalUncompressedFile(dataDir string) string { var ( extension string ) @@ -78,8 +81,8 @@ func (d Download) getLocalUncompressedName() string { case strings.HasSuffix(d.LocalPath, ".xz"): extension = ".xz" } - uncompressedFilename := filepath.Join(filepath.Dir(d.LocalPath), d.VMName+"_"+d.ImageName) - return strings.TrimSuffix(uncompressedFilename, extension) + uncompressedFilename := d.VMName + "_" + d.ImageName + return filepath.Join(dataDir, strings.TrimSuffix(uncompressedFilename, extension)) } func (g GenericDownload) Get() *Download { @@ -91,6 +94,18 @@ func (g GenericDownload) HasUsableCache() (bool, error) { return g.URL == nil, nil } +// CleanCache cleans out downloaded uncompressed image files +func (g GenericDownload) CleanCache() error { + // Remove any image that has been downloaded via URL + // We never read from cache for generic downloads + if g.URL != nil { + if err := os.Remove(g.LocalPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + return nil +} + func DownloadImage(d DistributionDownload) error { // check if the latest image is already present ok, err := d.HasUsableCache() @@ -101,8 +116,14 @@ func DownloadImage(d DistributionDownload) error { if err := DownloadVMImage(d.Get().URL, d.Get().LocalPath); err != nil { return err } + // Clean out old cached images, since we didn't find needed image in cache + defer func() { + if err = d.CleanCache(); err != nil { + logrus.Warnf("error cleaning machine image cache: %s", err) + } + }() } - return Decompress(d.Get().LocalPath, d.Get().getLocalUncompressedName()) + return Decompress(d.Get().LocalPath, d.Get().LocalUncompressedFile) } // DownloadVMImage downloads a VM image from url to given path @@ -253,3 +274,20 @@ func decompressEverythingElse(src string, output io.WriteCloser) error { _, err = io.Copy(output, uncompressStream) return err } + +func removeImageAfterExpire(dir string, expire time.Duration) error { + now := time.Now() + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // Delete any cache files that are older than expiry date + if !info.IsDir() && (now.Sub(info.ModTime()) > expire) { + err := os.Remove(path) + if err != nil && !errors.Is(err, os.ErrNotExist) { + logrus.Warnf("unable to clean up cached image: %s", path) + } else { + logrus.Debugf("cleaning up cached image: %s", path) + } + } + return nil + }) + return err +} -- cgit v1.2.3-54-g00ecf From d2e2756a4cf036c30dc1b9d804c4a3ed331efd5b Mon Sep 17 00:00:00 2001 From: "Jason T. Greene" Date: Mon, 18 Jul 2022 10:34:42 -0500 Subject: Print rootfs download as a specific version on Win - Also save the file using this convention. - Change the general pull mechanism to print the local file as opposed to the remote to enable this - no change in observed behavior on mac Signed-off-by: Jason T. Greene --- pkg/machine/fedora.go | 32 +++++++++++++++++++++++++------- pkg/machine/pull.go | 7 +++---- 2 files changed, 28 insertions(+), 11 deletions(-) (limited to 'pkg/machine/pull.go') diff --git a/pkg/machine/fedora.go b/pkg/machine/fedora.go index 7c80fc5d3..7ac4692e3 100644 --- a/pkg/machine/fedora.go +++ b/pkg/machine/fedora.go @@ -6,7 +6,10 @@ package machine import ( "errors" "fmt" + "io" "os" + "path" + "strings" "net/http" "net/url" @@ -23,7 +26,7 @@ type FedoraDownload struct { } func NewFedoraDownloader(vmType, vmName, releaseStream string) (DistributionDownload, error) { - downloadURL, size, err := getFedoraDownload(githubLatestReleaseURL) + downloadURL, version, size, err := getFedoraDownload(githubLatestReleaseURL) if err != nil { return nil, err } @@ -33,7 +36,7 @@ func NewFedoraDownloader(vmType, vmName, releaseStream string) (DistributionDown return nil, err } - imageName := "rootfs.tar.xz" + imageName := fmt.Sprintf("fedora-podman-%s.tar.xz", version) f := FedoraDownload{ Download: Download{ @@ -77,21 +80,36 @@ func (f FedoraDownload) CleanCache() error { return removeImageAfterExpire(f.CacheDir, expire) } -func getFedoraDownload(releaseURL string) (*url.URL, int64, error) { +func getFedoraDownload(releaseURL string) (*url.URL, string, int64, error) { downloadURL, err := url.Parse(releaseURL) if err != nil { - return nil, -1, fmt.Errorf("invalid URL generated from discovered Fedora file: %s: %w", releaseURL, err) + return nil, "", -1, fmt.Errorf("invalid URL generated from discovered Fedora file: %s: %w", releaseURL, err) } resp, err := http.Head(releaseURL) if err != nil { - return nil, -1, fmt.Errorf("head request failed: %s: %w", releaseURL, err) + return nil, "", -1, fmt.Errorf("head request failed: %s: %w", releaseURL, err) } _ = resp.Body.Close() + contentLen := resp.ContentLength if resp.StatusCode != http.StatusOK { - return nil, -1, fmt.Errorf("head request failed: %s: %w", releaseURL, err) + return nil, "", -1, fmt.Errorf("head request failed: %s: %w", releaseURL, err) } - return downloadURL, resp.ContentLength, nil + verURL := *downloadURL + verURL.Path = path.Join(path.Dir(downloadURL.Path), "version") + + resp, err = http.Get(verURL.String()) + if err != nil { + return nil, "", -1, fmt.Errorf("get request failed: %s: %w", verURL.String(), err) + } + + bytes, err := io.ReadAll(&io.LimitedReader{R: resp.Body, N: 1024}) + if err != nil { + return nil, "", -1, fmt.Errorf("failed reading: %s: %w", verURL.String(), err) + } + _ = resp.Body.Close() + + return downloadURL, strings.TrimSpace(string(bytes)), contentLen, nil } diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go index 08baa7df8..26b6adc67 100644 --- a/pkg/machine/pull.go +++ b/pkg/machine/pull.go @@ -113,7 +113,7 @@ func DownloadImage(d DistributionDownload) error { return err } if !ok { - if err := DownloadVMImage(d.Get().URL, d.Get().LocalPath); err != nil { + if err := DownloadVMImage(d.Get().URL, d.Get().ImageName, d.Get().LocalPath); err != nil { return err } // Clean out old cached images, since we didn't find needed image in cache @@ -128,7 +128,7 @@ func DownloadImage(d DistributionDownload) error { // DownloadVMImage downloads a VM image from url to given path // with download status -func DownloadVMImage(downloadURL *url2.URL, localImagePath string) error { +func DownloadVMImage(downloadURL *url2.URL, imageName string, localImagePath string) error { out, err := os.Create(localImagePath) if err != nil { return err @@ -153,8 +153,7 @@ func DownloadVMImage(downloadURL *url2.URL, localImagePath string) error { return fmt.Errorf("downloading VM image %s: %s", downloadURL, resp.Status) } size := resp.ContentLength - urlSplit := strings.Split(downloadURL.Path, "/") - prefix := "Downloading VM image: " + urlSplit[len(urlSplit)-1] + prefix := "Downloading VM image: " + imageName onComplete := prefix + ": done" p := mpb.New( -- cgit v1.2.3-54-g00ecf