Add a mutex to BoltDB state to prevent lock issues

Per https://www.sqlite.org/src/artifact/c230a7a24?ln=994-1081,
POSIX file advisory locks are unsafe to use within a single
process if multiple file descriptors are open for the same file.
Unfortunately, this has a strong potential to happen for
multithreaded usage of libpod, and could result in DB corruption.

To prevent this, wrap all access to BoltDB within a single
libpod instance in a mutex to ensure concurrent access cannot
occur.

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

1 files changed, 20 insertions, 1 deletions

diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go
index b03c11531..4687ed697 100644
--- a/libpod/boltdb_state_internal.go
+++ b/libpod/boltdb_state_internal.go
@@ -145,7 +145,15 @@ func validateDBAgainstConfig(bucket *bolt.Bucket, fieldName, runtimeValue string
 	return nil
 }
 
+// Open a connection to the database.
+// Must be paired with a `defer closeDBCon()` on the returned database, to
+// ensure the state is properly unlocked
 func (s *BoltState) getDBCon() (*bolt.DB, error) {
+	// We need an in-memory lock to avoid issues around POSIX file advisory
+	// locks as described in the link below:
+	// https://www.sqlite.org/src/artifact/c230a7a24?ln=994-1081
+	s.dbLock.Lock()
+
 	db, err := bolt.Open(s.dbPath, 0600, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error opening database %s", s.dbPath)
@@ -154,6 +162,17 @@ func (s *BoltState) getDBCon() (*bolt.DB, error) {
 	return db, nil
 }
 
+// Close a connection to the database.
+// MUST be used in place of `db.Close()` to ensure proper unlocking of the
+// state.
+func (s *BoltState) closeDBCon(db *bolt.DB) error {
+	err := db.Close()
+
+	s.dbLock.Unlock()
+
+	return err
+}
+
 func getIDBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 	bkt := tx.Bucket(idRegistryBkt)
 	if bkt == nil {
@@ -296,7 +315,7 @@ func (s *BoltState) addContainer(ctr *Container, pod *Pod) error {
 	if err != nil {
 		return err
 	}
-	defer db.Close()
+	defer s.closeDBCon(db)
 
 	err = db.Update(func(tx *bolt.Tx) error {
 		idsBucket, err := getIDBucket(tx)