Implement Secrets

Implement podman secret create, inspect, ls, rm
Implement podman run/create --secret
Secrets are blobs of data that are sensitive.
Currently, the only secret driver supported is filedriver, which means creating a secret stores it in base64 unencrypted in a file.
After creating a secret, a user can use the --secret flag to expose the secret inside the container at /run/secrets/[secretname]
This secret will not be commited to an image on a podman commit

Signed-off-by: Ashley Cui <acui@redhat.com>

4 files changed, 653 insertions, 0 deletions

diff --git a/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go b/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go
new file mode 100644
index 000000000..37edc16be
--- /dev/null
+++ b/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go
@@ -0,0 +1,158 @@
+package filedriver
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/containers/storage/pkg/lockfile"
+	"github.com/pkg/errors"
+)
+
+// secretsDataFile is the file where secrets data/payload will be stored
+var secretsDataFile = "secretsdata.json"
+
+// errNoSecretData indicates that there is not data associated with an id
+var errNoSecretData = errors.New("no secret data with ID")
+
+// errNoSecretData indicates that there is secret data already associated with an id
+var errSecretIDExists = errors.New("secret data with ID already exists")
+
+// Driver is the filedriver object
+type Driver struct {
+	// secretsDataFilePath is the path to the secretsfile
+	secretsDataFilePath string
+	// lockfile is the filedriver lockfile
+	lockfile lockfile.Locker
+}
+
+// NewDriver creates a new file driver.
+// rootPath is the directory where the secrets data file resides.
+func NewDriver(rootPath string) (*Driver, error) {
+	fileDriver := new(Driver)
+	fileDriver.secretsDataFilePath = filepath.Join(rootPath, secretsDataFile)
+	// the lockfile functions requre that the rootPath dir is executable
+	if err := os.MkdirAll(rootPath, 0700); err != nil {
+		return nil, err
+	}
+
+	lock, err := lockfile.GetLockfile(filepath.Join(rootPath, "secretsdata.lock"))
+	if err != nil {
+		return nil, err
+	}
+	fileDriver.lockfile = lock
+
+	return fileDriver, nil
+}
+
+// List returns all secret IDs
+func (d *Driver) List() ([]string, error) {
+	d.lockfile.Lock()
+	defer d.lockfile.Unlock()
+	secretData, err := d.getAllData()
+	if err != nil {
+		return nil, err
+	}
+	var allID []string
+	for k := range secretData {
+		allID = append(allID, k)
+	}
+	sort.Strings(allID)
+	return allID, err
+}
+
+// Lookup returns the bytes associated with a secret ID
+func (d *Driver) Lookup(id string) ([]byte, error) {
+	d.lockfile.Lock()
+	defer d.lockfile.Unlock()
+
+	secretData, err := d.getAllData()
+	if err != nil {
+		return nil, err
+	}
+	if data, ok := secretData[id]; ok {
+		return data, nil
+	}
+	return nil, errors.Wrapf(errNoSecretData, "%s", id)
+}
+
+// Store stores the bytes associated with an ID. An error is returned if the ID arleady exists
+func (d *Driver) Store(id string, data []byte) error {
+	d.lockfile.Lock()
+	defer d.lockfile.Unlock()
+
+	secretData, err := d.getAllData()
+	if err != nil {
+		return err
+	}
+	if _, ok := secretData[id]; ok {
+		return errors.Wrapf(errSecretIDExists, "%s", id)
+	}
+	secretData[id] = data
+	marshalled, err := json.MarshalIndent(secretData, "", "  ")
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0600)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Delete deletes the secret associated with the specified ID.  An error is returned if no matching secret is found.
+func (d *Driver) Delete(id string) error {
+	d.lockfile.Lock()
+	defer d.lockfile.Unlock()
+	secretData, err := d.getAllData()
+	if err != nil {
+		return err
+	}
+	if _, ok := secretData[id]; ok {
+		delete(secretData, id)
+	} else {
+		return errors.Wrap(errNoSecretData, id)
+	}
+	marshalled, err := json.MarshalIndent(secretData, "", "  ")
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0600)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// getAllData reads the data file and returns all data
+func (d *Driver) getAllData() (map[string][]byte, error) {
+	// check if the db file exists
+	_, err := os.Stat(d.secretsDataFilePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// the file will be created later on a store()
+			return make(map[string][]byte), nil
+		} else {
+			return nil, err
+		}
+	}
+
+	file, err := os.Open(d.secretsDataFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	byteValue, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+	secretData := new(map[string][]byte)
+	err = json.Unmarshal([]byte(byteValue), secretData)
+	if err != nil {
+		return nil, err
+	}
+	return *secretData, nil
+}
diff --git a/vendor/github.com/containers/common/pkg/secrets/secrets.go b/vendor/github.com/containers/common/pkg/secrets/secrets.go
new file mode 100644
index 000000000..5e0fb3e9d
--- /dev/null
+++ b/vendor/github.com/containers/common/pkg/secrets/secrets.go
@@ -0,0 +1,282 @@
+package secrets
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/containers/common/pkg/secrets/filedriver"
+	"github.com/containers/storage/pkg/lockfile"
+	"github.com/containers/storage/pkg/stringid"
+	"github.com/pkg/errors"
+)
+
+// maxSecretSize is the max size for secret data - 512kB
+const maxSecretSize = 512000
+
+// secretIDLength is the character length of a secret ID - 25
+const secretIDLength = 25
+
+// errInvalidPath indicates that the secrets path is invalid
+var errInvalidPath = errors.New("invalid secrets path")
+
+// errNoSuchSecret indicates that the secret does not exist
+var errNoSuchSecret = errors.New("no such secret")
+
+// errSecretNameInUse indicates that the secret name is already in use
+var errSecretNameInUse = errors.New("secret name in use")
+
+// errInvalidSecretName indicates that the secret name is invalid
+var errInvalidSecretName = errors.New("invalid secret name")
+
+// errInvalidDriver indicates that the driver type is invalid
+var errInvalidDriver = errors.New("invalid driver")
+
+// errInvalidDriverOpt indicates that a driver option is invalid
+var errInvalidDriverOpt = errors.New("invalid driver option")
+
+// errAmbiguous indicates that a secret is ambiguous
+var errAmbiguous = errors.New("secret is ambiguous")
+
+// errDataSize indicates that the secret data is too large or too small
+var errDataSize = errors.New("secret data must be larger than 0 and less than 512000 bytes")
+
+// secretsFile is the name of the file that the secrets database will be stored in
+var secretsFile = "secrets.json"
+
+// secretNameRegexp matches valid secret names
+// Allowed: 64 [a-zA-Z0-9-_.] characters, and the start and end character must be [a-zA-Z0-9]
+var secretNameRegexp = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`)
+
+// SecretsManager holds information on handling secrets
+type SecretsManager struct {
+	// secretsPath is the path to the db file where secrets are stored
+	secretsDBPath string
+	// lockfile is the locker for the secrets file
+	lockfile lockfile.Locker
+	// db is an in-memory cache of the database of secrets
+	db *db
+}
+
+// Secret defines a secret
+type Secret struct {
+	// Name is the name of the secret
+	Name string `json:"name"`
+	// ID is the unique secret ID
+	ID string `json:"id"`
+	// Metadata stores other metadata on the secret
+	Metadata map[string]string `json:"metadata,omitempty"`
+	// CreatedAt is when the secret was created
+	CreatedAt time.Time `json:"createdAt"`
+	// Driver is the driver used to store secret data
+	Driver string `json:"driver"`
+	// DriverOptions is other metadata needed to use the driver
+	DriverOptions map[string]string `json:"driverOptions"`
+}
+
+// SecretsDriver interfaces with the secrets data store.
+// The driver stores the actual bytes of secret data, as opposed to
+// the secret metadata.
+// Currently only the unencrypted filedriver is implemented.
+type SecretsDriver interface {
+	// List lists all secret ids in the secrets data store
+	List() ([]string, error)
+	// Lookup gets the secret's data bytes
+	Lookup(id string) ([]byte, error)
+	// Store stores the secret's data bytes
+	Store(id string, data []byte) error
+	// Delete deletes a secret's data from the driver
+	Delete(id string) error
+}
+
+// NewManager creates a new secrets manager
+// rootPath is the directory where the secrets data file resides
+func NewManager(rootPath string) (*SecretsManager, error) {
+	manager := new(SecretsManager)
+
+	if !filepath.IsAbs(rootPath) {
+		return nil, errors.Wrapf(errInvalidPath, "path must be absolute: %s", rootPath)
+	}
+	// the lockfile functions requre that the rootPath dir is executable
+	if err := os.MkdirAll(rootPath, 0700); err != nil {
+		return nil, err
+	}
+
+	lock, err := lockfile.GetLockfile(filepath.Join(rootPath, "secrets.lock"))
+	if err != nil {
+		return nil, err
+	}
+	manager.lockfile = lock
+	manager.secretsDBPath = filepath.Join(rootPath, secretsFile)
+	manager.db = new(db)
+	manager.db.Secrets = make(map[string]Secret)
+	manager.db.NameToID = make(map[string]string)
+	manager.db.IDToName = make(map[string]string)
+	return manager, nil
+}
+
+// Store takes a name, creates a secret and stores the secret metadata and the secret payload.
+// It returns a generated ID that is associated with the secret.
+// The max size for secret data is 512kB.
+func (s *SecretsManager) Store(name string, data []byte, driverType string, driverOpts map[string]string) (string, error) {
+	err := validateSecretName(name)
+	if err != nil {
+		return "", err
+	}
+
+	if !(len(data) > 0 && len(data) < maxSecretSize) {
+		return "", errDataSize
+	}
+
+	s.lockfile.Lock()
+	defer s.lockfile.Unlock()
+
+	exist, err := s.exactSecretExists(name)
+	if err != nil {
+		return "", err
+	}
+	if exist {
+		return "", errors.Wrapf(errSecretNameInUse, name)
+	}
+
+	secr := new(Secret)
+	secr.Name = name
+
+	for {
+		newID := stringid.GenerateNonCryptoID()
+		// GenerateNonCryptoID() gives 64 characters, so we truncate to correct length
+		newID = newID[0:secretIDLength]
+		_, err := s.lookupSecret(newID)
+		if err != nil {
+			if errors.Cause(err) == errNoSuchSecret {
+				secr.ID = newID
+				break
+			} else {
+				return "", err
+			}
+		}
+	}
+
+	secr.Driver = driverType
+	secr.Metadata = make(map[string]string)
+	secr.CreatedAt = time.Now()
+	secr.DriverOptions = driverOpts
+
+	driver, err := getDriver(driverType, driverOpts)
+	if err != nil {
+		return "", err
+	}
+	err = driver.Store(secr.ID, data)
+	if err != nil {
+		return "", errors.Wrapf(err, "error creating secret %s", name)
+	}
+
+	err = s.store(secr)
+	if err != nil {
+		return "", errors.Wrapf(err, "error creating secret %s", name)
+	}
+
+	return secr.ID, nil
+}
+
+// Delete removes all secret metadata and secret data associated with the specified secret.
+// Delete takes a name, ID, or partial ID.
+func (s *SecretsManager) Delete(nameOrID string) (string, error) {
+	err := validateSecretName(nameOrID)
+	if err != nil {
+		return "", err
+	}
+
+	s.lockfile.Lock()
+	defer s.lockfile.Unlock()
+
+	secret, err := s.lookupSecret(nameOrID)
+	if err != nil {
+		return "", err
+	}
+	secretID := secret.ID
+
+	driver, err := getDriver(secret.Driver, secret.DriverOptions)
+	if err != nil {
+		return "", err
+	}
+
+	err = driver.Delete(secretID)
+	if err != nil {
+		return "", errors.Wrapf(err, "error deleting secret %s", nameOrID)
+	}
+
+	err = s.delete(secretID)
+	if err != nil {
+		return "", errors.Wrapf(err, "error deleting secret %s", nameOrID)
+	}
+	return secretID, nil
+}
+
+// Lookup gives a secret's metadata given its name, ID, or partial ID.
+func (s *SecretsManager) Lookup(nameOrID string) (*Secret, error) {
+	s.lockfile.Lock()
+	defer s.lockfile.Unlock()
+
+	return s.lookupSecret(nameOrID)
+}
+
+// List lists all secrets.
+func (s *SecretsManager) List() ([]Secret, error) {
+	s.lockfile.Lock()
+	defer s.lockfile.Unlock()
+
+	secrets, err := s.lookupAll()
+	if err != nil {
+		return nil, err
+	}
+	var ls []Secret
+	for _, v := range secrets {
+		ls = append(ls, v)
+
+	}
+	return ls, nil
+}
+
+// LookupSecretData returns secret metadata as well as secret data in bytes.
+// The secret data can be looked up using its name, ID, or partial ID.
+func (s *SecretsManager) LookupSecretData(nameOrID string) (*Secret, []byte, error) {
+	s.lockfile.Lock()
+	defer s.lockfile.Unlock()
+
+	secret, err := s.lookupSecret(nameOrID)
+	if err != nil {
+		return nil, nil, err
+	}
+	driver, err := getDriver(secret.Driver, secret.DriverOptions)
+	if err != nil {
+		return nil, nil, err
+	}
+	data, err := driver.Lookup(secret.ID)
+	if err != nil {
+		return nil, nil, err
+	}
+	return secret, data, nil
+}
+
+// validateSecretName checks if the secret name is valid.
+func validateSecretName(name string) error {
+	if !secretNameRegexp.MatchString(name) || len(name) > 64 || strings.HasSuffix(name, "-") || strings.HasSuffix(name, ".") {
+		return errors.Wrapf(errInvalidSecretName, "only 64 [a-zA-Z0-9-_.] characters allowed, and the start and end character must be [a-zA-Z0-9]: %s", name)
+	}
+	return nil
+}
+
+// getDriver creates a new driver.
+func getDriver(name string, opts map[string]string) (SecretsDriver, error) {
+	if name == "file" {
+		if path, ok := opts["path"]; ok {
+			return filedriver.NewDriver(path)
+		} else {
+			return nil, errors.Wrap(errInvalidDriverOpt, "need path for filedriver")
+		}
+	}
+	return nil, errInvalidDriver
+}
diff --git a/vendor/github.com/containers/common/pkg/secrets/secretsdb.go b/vendor/github.com/containers/common/pkg/secrets/secretsdb.go
new file mode 100644
index 000000000..22db97c12
--- /dev/null
+++ b/vendor/github.com/containers/common/pkg/secrets/secretsdb.go
@@ -0,0 +1,211 @@
+package secrets
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+type db struct {
+	// Secrets maps a secret id to secret metadata
+	Secrets map[string]Secret `json:"secrets"`
+	// NameToID maps a secret name to a secret id
+	NameToID map[string]string `json:"nameToID"`
+	// IDToName maps a secret id to a secret name
+	IDToName map[string]string `json:"idToName"`
+	// lastModified is the time when the database was last modified on the file system
+	lastModified time.Time
+}
+
+// loadDB loads database data into the in-memory cache if it has been modified
+func (s *SecretsManager) loadDB() error {
+	// check if the db file exists
+	fileInfo, err := os.Stat(s.secretsDBPath)
+	if err != nil {
+		if !os.IsExist(err) {
+			// If the file doesn't exist, then there's no reason to update the db cache,
+			// the db cache will show no entries anyway.
+			// The file will be created later on a store()
+			return nil
+		} else {
+			return err
+		}
+	}
+
+	// We check if the file has been modified after the last time it was loaded into the cache.
+	// If the file has been modified, then we know that our cache is not up-to-date, so we load
+	// the db into the cache.
+	if s.db.lastModified.Equal(fileInfo.ModTime()) {
+		return nil
+	}
+
+	file, err := os.Open(s.secretsDBPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if err != nil {
+		return err
+	}
+
+	byteValue, err := ioutil.ReadAll(file)
+	if err != nil {
+		return err
+	}
+	unmarshalled := new(db)
+	if err := json.Unmarshal(byteValue, unmarshalled); err != nil {
+		return err
+	}
+	s.db = unmarshalled
+	s.db.lastModified = fileInfo.ModTime()
+
+	return nil
+}
+
+// getNameAndID takes a secret's name, ID, or partial ID, and returns both its name and full ID.
+func (s *SecretsManager) getNameAndID(nameOrID string) (name, id string, err error) {
+	name, id, err = s.getExactNameAndID(nameOrID)
+	if err == nil {
+		return name, id, nil
+	} else if errors.Cause(err) != errNoSuchSecret {
+		return "", "", err
+	}
+
+	// ID prefix may have been given, iterate through all IDs.
+	// ID and partial ID has a max lenth of 25, so we return if its greater than that.
+	if len(nameOrID) > secretIDLength {
+		return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+	}
+	exists := false
+	var foundID, foundName string
+	for id, name := range s.db.IDToName {
+		if strings.HasPrefix(id, nameOrID) {
+			if exists {
+				return "", "", errors.Wrapf(errAmbiguous, "more than one result secret with prefix %s", nameOrID)
+			}
+			exists = true
+			foundID = id
+			foundName = name
+		}
+	}
+
+	if exists {
+		return foundName, foundID, nil
+	}
+	return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+}
+
+// getExactNameAndID takes a secret's name or ID and returns both its name and full ID.
+func (s *SecretsManager) getExactNameAndID(nameOrID string) (name, id string, err error) {
+	err = s.loadDB()
+	if err != nil {
+		return "", "", err
+	}
+	if name, ok := s.db.IDToName[nameOrID]; ok {
+		id := nameOrID
+		return name, id, nil
+	}
+
+	if id, ok := s.db.NameToID[nameOrID]; ok {
+		name := nameOrID
+		return name, id, nil
+	}
+
+	return "", "", errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+}
+
+// exactSecretExists checks if the secret exists, given a name or ID
+// Does not match partial name or IDs
+func (s *SecretsManager) exactSecretExists(nameOrID string) (bool, error) {
+	_, _, err := s.getExactNameAndID(nameOrID)
+	if err != nil {
+		if errors.Cause(err) == errNoSuchSecret {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// lookupAll gets all secrets stored.
+func (s *SecretsManager) lookupAll() (map[string]Secret, error) {
+	err := s.loadDB()
+	if err != nil {
+		return nil, err
+	}
+	return s.db.Secrets, nil
+}
+
+// lookupSecret returns a secret with the given name, ID, or partial ID.
+func (s *SecretsManager) lookupSecret(nameOrID string) (*Secret, error) {
+	err := s.loadDB()
+	if err != nil {
+		return nil, err
+	}
+	_, id, err := s.getNameAndID(nameOrID)
+	if err != nil {
+		return nil, err
+	}
+	allSecrets, err := s.lookupAll()
+	if err != nil {
+		return nil, err
+	}
+	if secret, ok := allSecrets[id]; ok {
+		return &secret, nil
+	}
+
+	return nil, errors.Wrapf(errNoSuchSecret, "no secret with name or id %q", nameOrID)
+}
+
+// Store creates a new secret in the secrets database.
+// It deals with only storing metadata, not data payload.
+func (s *SecretsManager) store(entry *Secret) error {
+	err := s.loadDB()
+	if err != nil {
+		return err
+	}
+
+	s.db.Secrets[entry.ID] = *entry
+	s.db.NameToID[entry.Name] = entry.ID
+	s.db.IDToName[entry.ID] = entry.Name
+
+	marshalled, err := json.MarshalIndent(s.db, "", "  ")
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0600)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// delete deletes a secret from the secrets database, given a name, ID, or partial ID.
+// It deals with only deleting metadata, not data payload.
+func (s *SecretsManager) delete(nameOrID string) error {
+	name, id, err := s.getNameAndID(nameOrID)
+	if err != nil {
+		return err
+	}
+	err = s.loadDB()
+	if err != nil {
+		return err
+	}
+	delete(s.db.Secrets, id)
+	delete(s.db.NameToID, name)
+	delete(s.db.IDToName, id)
+	marshalled, err := json.MarshalIndent(s.db, "", "  ")
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0600)
+	if err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index e8b5edf8c..6c425ac06 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -102,6 +102,8 @@ github.com/containers/common/pkg/report
 github.com/containers/common/pkg/report/camelcase
 github.com/containers/common/pkg/retry
 github.com/containers/common/pkg/seccomp
+github.com/containers/common/pkg/secrets
+github.com/containers/common/pkg/secrets/filedriver
 github.com/containers/common/pkg/subscriptions
 github.com/containers/common/pkg/sysinfo
 github.com/containers/common/pkg/umask