From 185136cf0edee5288576b541517e0e994f6ee18d Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 8 Aug 2018 11:22:44 -0400 Subject: Add interface for libpod multiprocess locks Signed-off-by: Matthew Heon --- libpod/lock/lock.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 libpod/lock/lock.go (limited to 'libpod/lock/lock.go') diff --git a/libpod/lock/lock.go b/libpod/lock/lock.go new file mode 100644 index 000000000..6999e1118 --- /dev/null +++ b/libpod/lock/lock.go @@ -0,0 +1,55 @@ +package lock + +// LockManager provides an interface for allocating multiprocess locks. +// Locks returned by LockManager MUST be multiprocess - allocating a lock in +// process A and retrieving that lock's ID in process B must return handles for +// the same lock, and locking the lock in A should exclude B from the lock until +// it is unlocked in A. +// All locks must be identified by a UUID (retrieved with Locker's ID() method). +// All locks with a given UUID must refer to the same underlying lock, and it +// must be possible to retrieve the lock given its UUID. +// Each UUID should refer to a unique underlying lock. +// Calls to AllocateLock() must return a unique, unallocated UUID. +// AllocateLock() must fail once all available locks have been allocated. +// Locks are returned to use by calls to Free(), and can subsequently be +// reallocated. +type LockManager interface { + // AllocateLock returns an unallocated lock. + // It is guaranteed that the same lock will not be returned again by + // AllocateLock until the returned lock has Free() called on it. + // If all available locks are allocated, AllocateLock will return an + // error. + AllocateLock() (Locker, error) + // RetrieveLock retrieves a lock given its UUID. + // The underlying lock MUST be the same as another other lock with the + // same UUID. + RetrieveLock(id string) (Locker, error) +} + +// Locker is similar to sync.Locker, but provides a method for freeing the lock +// to allow its reuse. +// All Locker implementations must maintain mutex semantics - the lock only +// allows one caller in the critical section at a time. +// All locks with the same ID must refer to the same underlying lock, even +// if they are within multiple processes. +type Locker interface { + // ID retrieves the lock's ID. + // ID is guaranteed to uniquely identify the lock within the + // LockManager - that is, calling RetrieveLock with this ID will return + // another instance of the same lock. + ID() string + // Lock locks the lock. + // This call MUST block until it successfully acquires the lock or + // encounters a fatal error. + Lock() error + // Unlock unlocks the lock. + // A call to Unlock() on a lock that is already unlocked lock MUST + // error. + Unlock() error + // Deallocate deallocates the underlying lock, allowing its reuse by + // other pods and containers. + // The lock MUST still be usable after a Free() - some libpod instances + // may still retain Container structs with the old lock. This simply + // advises the manager that the lock may be reallocated. + Free() error +} -- cgit v1.2.3-54-g00ecf From 3ed81051e814e688f7b4ae1bbc7c4d4c3fbd7d0f Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 8 Aug 2018 12:26:01 -0400 Subject: Add an SHM-backed Lock Manager implementation Signed-off-by: Matthew Heon --- libpod/lock/lock.go | 8 +-- libpod/lock/shm_lock.go | 8 +-- libpod/lock/shm_lock_manager.go | 109 ++++++++++++++++++++++++++++++++++++++++ libpod/lock/shm_lock_test.go | 5 +- 4 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 libpod/lock/shm_lock_manager.go (limited to 'libpod/lock/lock.go') diff --git a/libpod/lock/lock.go b/libpod/lock/lock.go index 6999e1118..6d17828f6 100644 --- a/libpod/lock/lock.go +++ b/libpod/lock/lock.go @@ -1,7 +1,7 @@ package lock -// LockManager provides an interface for allocating multiprocess locks. -// Locks returned by LockManager MUST be multiprocess - allocating a lock in +// Manager provides an interface for allocating multiprocess locks. +// Locks returned by Manager MUST be multiprocess - allocating a lock in // process A and retrieving that lock's ID in process B must return handles for // the same lock, and locking the lock in A should exclude B from the lock until // it is unlocked in A. @@ -13,7 +13,7 @@ package lock // AllocateLock() must fail once all available locks have been allocated. // Locks are returned to use by calls to Free(), and can subsequently be // reallocated. -type LockManager interface { +type Manager interface { // AllocateLock returns an unallocated lock. // It is guaranteed that the same lock will not be returned again by // AllocateLock until the returned lock has Free() called on it. @@ -35,7 +35,7 @@ type LockManager interface { type Locker interface { // ID retrieves the lock's ID. // ID is guaranteed to uniquely identify the lock within the - // LockManager - that is, calling RetrieveLock with this ID will return + // Manager - that is, calling RetrieveLock with this ID will return // another instance of the same lock. ID() string // Lock locks the lock. diff --git a/libpod/lock/shm_lock.go b/libpod/lock/shm_lock.go index 4d7c26aa2..a8a969479 100644 --- a/libpod/lock/shm_lock.go +++ b/libpod/lock/shm_lock.go @@ -27,13 +27,13 @@ type SHMLocks struct { // semaphores, and returns a struct that can be used to operate on those locks. // numLocks must be a multiple of the lock bitmap size (by default, 32). func CreateSHMLock(numLocks uint32) (*SHMLocks, error) { - if numLocks % bitmapSize != 0 || numLocks == 0 { + if numLocks%bitmapSize != 0 || numLocks == 0 { return nil, errors.Wrapf(syscall.EINVAL, "number of locks must be a multiple of %d", C.bitmap_size_c) } locks := new(SHMLocks) - var errCode C.int = 0 + var errCode C.int lockStruct := C.setup_lock_shm(C.uint32_t(numLocks), &errCode) if lockStruct == nil { // We got a null pointer, so something errored @@ -52,13 +52,13 @@ func CreateSHMLock(numLocks uint32) (*SHMLocks, error) { // segment was created with and be a multiple of the lock bitmap size (default // 32). func OpenSHMLock(numLocks uint32) (*SHMLocks, error) { - if numLocks % bitmapSize != 0 || numLocks == 0 { + if numLocks%bitmapSize != 0 || numLocks == 0 { return nil, errors.Wrapf(syscall.EINVAL, "number of locks must be a multiple of %d", C.bitmap_size_c) } locks := new(SHMLocks) - var errCode C.int = 0 + var errCode C.int lockStruct := C.open_lock_shm(C.uint32_t(numLocks), &errCode) if lockStruct == nil { // We got a null pointer, so something errored diff --git a/libpod/lock/shm_lock_manager.go b/libpod/lock/shm_lock_manager.go new file mode 100644 index 000000000..1fc2b106e --- /dev/null +++ b/libpod/lock/shm_lock_manager.go @@ -0,0 +1,109 @@ +package lock + +import ( + "fmt" + "math" + "strconv" + "syscall" + + "github.com/pkg/errors" +) + +// SHMLockManager manages shared memory locks. +type SHMLockManager struct { + locks *SHMLocks +} + +// NewSHMLockManager makes a new SHMLockManager with the given number of locks. +func NewSHMLockManager(numLocks uint32) (Manager, error) { + locks, err := CreateSHMLock(numLocks) + if err != nil { + return nil, err + } + + manager := new(SHMLockManager) + manager.locks = locks + + return manager, nil +} + +// OpenSHMLockManager opens an existing SHMLockManager with the given number of +// locks. +func OpenSHMLockManager(numLocks uint32) (LockManager, error) { + locks, err := OpenSHMLock(numLocks) + if err != nil { + return nil, err + } + + manager := new(SHMLockManager) + manager.locks = locks + + return manager, nil +} + +// AllocateLock allocates a new lock from the manager. +func (m *SHMLockManager) AllocateLock() (Locker, error) { + semIndex, err := m.locks.AllocateSemaphore() + if err != nil { + return nil, err + } + + lock := new(SHMLock) + lock.lockID = semIndex + lock.manager = m + + return lock, nil +} + +// RetrieveLock retrieves a lock from the manager given its ID. +func (m *SHMLockManager) RetrieveLock(id string) (Locker, error) { + intID, err := strconv.ParseInt(id, 16, 64) + if err != nil { + return errors.Wrapf(err, "given ID %q is not a valid SHMLockManager ID - cannot be parsed as int", id) + } + + if intID < 0 { + return errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - must be positive", id) + } + + if intID > math.MaxUint32 { + return errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - too large", id) + } + + var u32ID uint32 = uint32(intID) + if u32ID >= m.locks.maxLocks { + return errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - too large to fit", id) + } + + lock := new(SHMLock) + lock.lockID = u32ID + lock.manager = m + + return lock, nil +} + +// SHMLock is an individual shared memory lock. +type SHMLock struct { + lockID uint32 + manager *SHMLockManager +} + +// ID returns the ID of the lock. +func (l *SHMLock) ID() string { + return fmt.Sprintf("%x", l.lockID) +} + +// Lock acquires the lock. +func (l *SHMLock) Lock() error { + return l.manager.locks.LockSemaphore(l.lockID) +} + +// Unlock releases the lock. +func (l *SHMLock) Unlock() error { + return l.manager.locks.UnlockSemaphore(l.lockID) +} + +// Free releases the lock, allowing it to be reused. +func (l *SHMLock) Free() error { + return l.manager.locks.DeallocateSemaphore(l.lockID) +} diff --git a/libpod/lock/shm_lock_test.go b/libpod/lock/shm_lock_test.go index 6d4525f6a..4903d3a50 100644 --- a/libpod/lock/shm_lock_test.go +++ b/libpod/lock/shm_lock_test.go @@ -4,8 +4,8 @@ import ( "fmt" "os" "syscall" - "time" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -41,7 +41,6 @@ func TestMain(m *testing.M) { os.Exit(exitCode) } - func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { locks, err := OpenSHMLock(numLocks) if err != nil { @@ -66,7 +65,7 @@ func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { } }() - success := t.Run("locks", func (t *testing.T) { + success := t.Run("locks", func(t *testing.T) { testFunc(t, locks) }) if !success { -- cgit v1.2.3-54-g00ecf From e73484c176839b2f2adf3d07cc09222a7b75bf69 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Fri, 10 Aug 2018 13:46:07 -0400 Subject: Move to POSIX mutexes for SHM locks Signed-off-by: Matthew Heon --- libpod/lock/lock.go | 8 +- libpod/lock/shm/shm_lock.c | 245 ++++++++++++++++++++++------------ libpod/lock/shm/shm_lock.go | 52 +++++--- libpod/lock/shm/shm_lock.h | 35 ++--- libpod/lock/shm/shm_lock_test.go | 56 ++++++-- libpod/lock/shm_lock_manager_linux.go | 42 ++---- 6 files changed, 269 insertions(+), 169 deletions(-) (limited to 'libpod/lock/lock.go') diff --git a/libpod/lock/lock.go b/libpod/lock/lock.go index 6d17828f6..5258c641f 100644 --- a/libpod/lock/lock.go +++ b/libpod/lock/lock.go @@ -23,7 +23,7 @@ type Manager interface { // RetrieveLock retrieves a lock given its UUID. // The underlying lock MUST be the same as another other lock with the // same UUID. - RetrieveLock(id string) (Locker, error) + RetrieveLock(id uint32) (Locker, error) } // Locker is similar to sync.Locker, but provides a method for freeing the lock @@ -37,7 +37,7 @@ type Locker interface { // ID is guaranteed to uniquely identify the lock within the // Manager - that is, calling RetrieveLock with this ID will return // another instance of the same lock. - ID() string + ID() uint32 // Lock locks the lock. // This call MUST block until it successfully acquires the lock or // encounters a fatal error. @@ -46,8 +46,8 @@ type Locker interface { // A call to Unlock() on a lock that is already unlocked lock MUST // error. Unlock() error - // Deallocate deallocates the underlying lock, allowing its reuse by - // other pods and containers. + // Free deallocates the underlying lock, allowing its reuse by other + // pods and containers. // The lock MUST still be usable after a Free() - some libpod instances // may still retain Container structs with the old lock. This simply // advises the manager that the lock may be reallocated. diff --git a/libpod/lock/shm/shm_lock.c b/libpod/lock/shm/shm_lock.c index 3fe41f63c..4af58d857 100644 --- a/libpod/lock/shm/shm_lock.c +++ b/libpod/lock/shm/shm_lock.c @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include @@ -12,19 +12,67 @@ #include "shm_lock.h" // Compute the size of the SHM struct -size_t compute_shm_size(uint32_t num_bitmaps) { +static size_t compute_shm_size(uint32_t num_bitmaps) { return sizeof(shm_struct_t) + (num_bitmaps * sizeof(lock_group_t)); } +// Take the given mutex. +// Handles exceptional conditions, including a mutex locked by a process that +// died holding it. +// Returns 0 on success, or positive errno on failure. +static int take_mutex(pthread_mutex_t *mutex) { + int ret_code; + + do { + ret_code = pthread_mutex_lock(mutex); + } while(ret_code == EAGAIN); + + if (ret_code == EOWNERDEAD) { + // The previous owner of the mutex died while holding it + // Take it for ourselves + ret_code = pthread_mutex_consistent(mutex); + if (ret_code != 0) { + // Someone else may have gotten here first and marked the state consistent + // However, the mutex could also be invalid. + // Fail here instead of looping back to trying to lock the mutex. + return ret_code; + } + } else if (ret_code != 0) { + return ret_code; + } + + return 0; +} + +// Release the given mutex. +// Returns 0 on success, or positive errno on failure. +static int release_mutex(pthread_mutex_t *mutex) { + int ret_code; + + do { + ret_code = pthread_mutex_unlock(mutex); + } while(ret_code == EAGAIN); + + if (ret_code != 0) { + return ret_code; + } + + return 0; +} + // Set up an SHM segment holding locks for libpod. -// num_locks must be a multiple of BITMAP_SIZE (32 by default). +// num_locks must not be 0. +// Path is the path to the SHM segment. It must begin with a single / and +// container no other / characters, and be at most 255 characters including +// terminating NULL byte. // Returns a valid pointer on success or NULL on error. -// If an error occurs, it will be written to the int pointed to by error_code. -shm_struct_t *setup_lock_shm(uint32_t num_locks, int *error_code) { +// If an error occurs, negative ERRNO values will be written to error_code. +shm_struct_t *setup_lock_shm(char *path, uint32_t num_locks, int *error_code) { int shm_fd, i, j, ret_code; uint32_t num_bitmaps; size_t shm_size; shm_struct_t *shm; + pthread_mutexattr_t attr; // If error_code doesn't point to anything, we can't reasonably return errors // So fail immediately @@ -34,67 +82,93 @@ shm_struct_t *setup_lock_shm(uint32_t num_locks, int *error_code) { // We need a nonzero number of locks if (num_locks == 0) { - *error_code = EINVAL; + *error_code = -1 * EINVAL; return NULL; } - // Calculate the number of bitmaps required - if (num_locks % BITMAP_SIZE != 0) { - // Number of locks not a multiple of BITMAP_SIZE - *error_code = EINVAL; + if (path == NULL) { + *error_code = -1 * EINVAL; return NULL; } + + // Calculate the number of bitmaps required num_bitmaps = num_locks / BITMAP_SIZE; + if (num_locks % BITMAP_SIZE != 0) { + // The actual number given is not an even multiple of our bitmap size + // So round up + num_bitmaps += 1; + } // Calculate size of the shm segment shm_size = compute_shm_size(num_bitmaps); // Create a new SHM segment for us - shm_fd = shm_open(SHM_NAME, O_RDWR | O_CREAT | O_EXCL, 0600); + shm_fd = shm_open(path, O_RDWR | O_CREAT | O_EXCL, 0600); if (shm_fd < 0) { - *error_code = errno; + *error_code = -1 * errno; return NULL; } // Increase its size to what we need ret_code = ftruncate(shm_fd, shm_size); if (ret_code < 0) { - *error_code = errno; + *error_code = -1 * errno; goto CLEANUP_UNLINK; } // Map the shared memory in shm = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0); if (shm == MAP_FAILED) { - *error_code = errno; + *error_code = -1 * errno; goto CLEANUP_UNLINK; } // We have successfully mapped the memory, now initialize the region shm->magic = MAGIC; - shm->num_locks = num_locks; + shm->unused = 0; + shm->num_locks = num_bitmaps * BITMAP_SIZE; shm->num_bitmaps = num_bitmaps; - // Initialize the semaphore that protects the bitmaps. - // Initialize to value 1, as we're a mutex, and set pshared as this will be - // shared between processes in an SHM. - ret_code = sem_init(&(shm->segment_lock), true, 1); - if (ret_code < 0) { - *error_code = errno; + // Create an initializer for our pthread mutexes + ret_code = pthread_mutexattr_init(&attr); + if (ret_code != 0) { + *error_code = -1 * ret_code; goto CLEANUP_UNMAP; } + // Set mutexes to pshared - multiprocess-safe + ret_code = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); + if (ret_code != 0) { + *error_code = -1 * ret_code; + goto CLEANUP_FREEATTR; + } + + // Set mutexes to robust - if a process dies while holding a mutex, we'll get + // a special error code on the next attempt to lock it. + // This should prevent panicing processes from leaving the state unusable. + ret_code = pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST); + if (ret_code != 0) { + *error_code = -1 * ret_code; + goto CLEANUP_FREEATTR; + } + + // Initialize the mutex that protects the bitmaps using the mutex attributes + ret_code = pthread_mutex_init(&(shm->segment_lock), &attr); + if (ret_code != 0) { + *error_code = -1 * ret_code; + goto CLEANUP_FREEATTR; + } + // Initialize all bitmaps to 0 initially // And initialize all semaphores they use for (i = 0; i < num_bitmaps; i++) { shm->locks[i].bitmap = 0; for (j = 0; j < BITMAP_SIZE; j++) { - // As above, initialize to 1 to act as a mutex, and set pshared as we'll - // be living in an SHM. - ret_code = sem_init(&(shm->locks[i].locks[j]), true, 1); - if (ret_code < 0) { - *error_code = errno; - goto CLEANUP_UNMAP; + // Initialize each mutex + ret_code = pthread_mutex_init(&(shm->locks[i].locks[j]), &attr); + if (ret_code != 0) { + *error_code = -1 * ret_code; + goto CLEANUP_FREEATTR; } } } @@ -103,23 +177,33 @@ shm_struct_t *setup_lock_shm(uint32_t num_locks, int *error_code) { // Ignore errors, it's ok if we leak a single FD and this should only run once close(shm_fd); + // Destroy the pthread initializer attribute. + // Again, ignore errors, this will only run once and we might leak a tiny bit + // of memory at worst. + pthread_mutexattr_destroy(&attr); + return shm; // Cleanup after an error + CLEANUP_FREEATTR: + pthread_mutexattr_destroy(&attr); CLEANUP_UNMAP: munmap(shm, shm_size); CLEANUP_UNLINK: close(shm_fd); - shm_unlink(SHM_NAME); + shm_unlink(path); return NULL; } // Open an existing SHM segment holding libpod locks. // num_locks is the number of locks that will be configured in the SHM segment. -// num_locks must be a multiple of BITMAP_SIZE (32 by default). +// num_locks cannot be 0. +// Path is the path to the SHM segment. It must begin with a single / and +// container no other / characters, and be at most 255 characters including +// terminating NULL byte. // Returns a valid pointer on success or NULL on error. -// If an error occurs, it will be written to the int pointed to by error_code. -shm_struct_t *open_lock_shm(uint32_t num_locks, int *error_code) { +// If an error occurs, negative ERRNO values will be written to error_code. +shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code) { int shm_fd; shm_struct_t *shm; size_t shm_size; @@ -131,30 +215,34 @@ shm_struct_t *open_lock_shm(uint32_t num_locks, int *error_code) { // We need a nonzero number of locks if (num_locks == 0) { - *error_code = EINVAL; + *error_code = -1 * EINVAL; return NULL; } - // Calculate the number of bitmaps required - if (num_locks % BITMAP_SIZE != 0) { - // Number of locks not a multiple of BITMAP_SIZE - *error_code = EINVAL; + if (path == NULL) { + *error_code = -1 * EINVAL; return NULL; } + + // Calculate the number of bitmaps required num_bitmaps = num_locks / BITMAP_SIZE; + if (num_locks % BITMAP_SIZE != 0) { + num_bitmaps += 1; + } // Calculate size of the shm segment shm_size = compute_shm_size(num_bitmaps); - shm_fd = shm_open(SHM_NAME, O_RDWR, 0600); + shm_fd = shm_open(path, O_RDWR, 0600); if (shm_fd < 0) { + *error_code = -1 * errno; return NULL; } // Map the shared memory in shm = mmap(NULL, shm_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0); if (shm == MAP_FAILED) { - *error_code = errno; + *error_code = -1 * errno; } // Ignore errors, it's ok if we leak a single FD since this only runs once @@ -167,11 +255,11 @@ shm_struct_t *open_lock_shm(uint32_t num_locks, int *error_code) { // Need to check the SHM to see if it's actually our locks if (shm->magic != MAGIC) { - *error_code = errno; + *error_code = -1 * errno; goto CLEANUP; } - if (shm->num_locks != num_locks) { - *error_code = errno; + if (shm->num_locks != (num_bitmaps * BITMAP_SIZE)) { + *error_code = -1 * errno; goto CLEANUP; } @@ -219,11 +307,9 @@ int64_t allocate_semaphore(shm_struct_t *shm) { } // Lock the semaphore controlling access to our shared memory - do { - ret_code = sem_wait(&(shm->segment_lock)); - } while(ret_code == EINTR); + ret_code = take_mutex(&(shm->segment_lock)); if (ret_code != 0) { - return -1 * errno; + return -1 * ret_code; } // Loop through our bitmaps to search for one that is not full @@ -237,8 +323,13 @@ int64_t allocate_semaphore(shm_struct_t *shm) { sem_number = (BITMAP_SIZE * i) + num_within_bitmap; // OR in the bitmap shm->locks[i].bitmap = shm->locks[i].bitmap | test_map; - // Clear the semaphore - sem_post(&(shm->segment_lock)); + + // Clear the mutex + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } + // Return the semaphore we've allocated return sem_number; } @@ -250,8 +341,11 @@ int64_t allocate_semaphore(shm_struct_t *shm) { } } - // Post to the semaphore to clear the lock - sem_post(&(shm->segment_lock)); + // Clear the mutex + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } // All bitmaps are full // We have no available semaphores, report allocation failure @@ -282,23 +376,20 @@ int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index) { return -1 * EFAULT; } - test_map = 0x1; - for (i = 0; i < index_in_bitmap; i++) { - test_map = test_map << 1; - } + test_map = 0x1 << index_in_bitmap; - // Lock the semaphore controlling access to our shared memory - do { - ret_code = sem_wait(&(shm->segment_lock)); - } while(ret_code == EINTR); + // Lock the mutex controlling access to our shared memory + ret_code = take_mutex(&(shm->segment_lock)); if (ret_code != 0) { - return -1 * errno; + return -1 * ret_code; } // Check if the semaphore is allocated if ((test_map & shm->locks[bitmap_index].bitmap) == 0) { - // Post to the semaphore to clear the lock - sem_post(&(shm->segment_lock)); + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } return -1 * ENOENT; } @@ -308,8 +399,10 @@ int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index) { test_map = ~test_map; shm->locks[bitmap_index].bitmap = shm->locks[bitmap_index].bitmap & test_map; - // Post to the semaphore to clear the lock - sem_post(&(shm->segment_lock)); + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } return 0; } @@ -333,15 +426,7 @@ int32_t lock_semaphore(shm_struct_t *shm, uint32_t sem_index) { bitmap_index = sem_index / BITMAP_SIZE; index_in_bitmap = sem_index % BITMAP_SIZE; - // Lock the semaphore controlling access to our shared memory - do { - ret_code = sem_wait(&(shm->locks[bitmap_index].locks[index_in_bitmap])); - } while(ret_code == EINTR); - if (ret_code != 0) { - return -1 * errno; - } - - return 0; + return -1 * take_mutex(&(shm->locks[bitmap_index].locks[index_in_bitmap])); } // Unlock a given semaphore @@ -351,7 +436,6 @@ int32_t lock_semaphore(shm_struct_t *shm, uint32_t sem_index) { // Returns 0 on success, -1 on failure int32_t unlock_semaphore(shm_struct_t *shm, uint32_t sem_index) { int bitmap_index, index_in_bitmap, ret_code; - unsigned int sem_value = 0; if (shm == NULL) { return -1 * EINVAL; @@ -364,20 +448,5 @@ int32_t unlock_semaphore(shm_struct_t *shm, uint32_t sem_index) { bitmap_index = sem_index / BITMAP_SIZE; index_in_bitmap = sem_index % BITMAP_SIZE; - // Only allow a post if the semaphore is less than 1 (locked) - // This allows us to preserve mutex behavior - ret_code = sem_getvalue(&(shm->locks[bitmap_index].locks[index_in_bitmap]), &sem_value); - if (ret_code != 0) { - return -1 * errno; - } - if (sem_value >= 1) { - return -1 * EBUSY; - } - - ret_code = sem_post(&(shm->locks[bitmap_index].locks[index_in_bitmap])); - if (ret_code != 0) { - return -1 * errno; - } - - return 0; + return -1 * release_mutex(&(shm->locks[bitmap_index].locks[index_in_bitmap])); } diff --git a/libpod/lock/shm/shm_lock.go b/libpod/lock/shm/shm_lock.go index 9a9074c04..16d7f2008 100644 --- a/libpod/lock/shm/shm_lock.go +++ b/libpod/lock/shm/shm_lock.go @@ -1,47 +1,54 @@ package shm // #cgo LDFLAGS: -lrt -lpthread +// #include // #include "shm_lock.h" // const uint32_t bitmap_size_c = BITMAP_SIZE; import "C" import ( + "runtime" "syscall" + "unsafe" "github.com/pkg/errors" ) -var ( - bitmapSize uint32 = uint32(C.bitmap_size_c) +const ( + BitmapSize uint32 = uint32(C.bitmap_size_c) ) // SHMLocks is a struct enabling POSIX semaphore locking in a shared memory // segment. type SHMLocks struct { // nolint lockStruct *C.shm_struct_t - valid bool maxLocks uint32 + valid bool } // CreateSHMLock sets up a shared-memory segment holding a given number of POSIX // semaphores, and returns a struct that can be used to operate on those locks. -// numLocks must be a multiple of the lock bitmap size (by default, 32). -func CreateSHMLock(numLocks uint32) (*SHMLocks, error) { - if numLocks%bitmapSize != 0 || numLocks == 0 { - return nil, errors.Wrapf(syscall.EINVAL, "number of locks must be a multiple of %d", C.bitmap_size_c) +// numLocks must not be 0, and may be rounded up to a multiple of the bitmap +// size used by the underlying implementation. +func CreateSHMLock(path string, numLocks uint32) (*SHMLocks, error) { + if numLocks == 0 { + return nil, errors.Wrapf(syscall.EINVAL, "number of locks must greater than 0 0") } locks := new(SHMLocks) + cPath := C.CString(path) + defer C.free(unsafe.Pointer(cPath)) + var errCode C.int - lockStruct := C.setup_lock_shm(C.uint32_t(numLocks), &errCode) + lockStruct := C.setup_lock_shm(cPath, C.uint32_t(numLocks), &errCode) if lockStruct == nil { // We got a null pointer, so something errored return nil, syscall.Errno(-1 * errCode) } locks.lockStruct = lockStruct - locks.maxLocks = numLocks + locks.maxLocks = uint32(lockStruct.num_locks) locks.valid = true return locks, nil @@ -49,17 +56,19 @@ func CreateSHMLock(numLocks uint32) (*SHMLocks, error) { // OpenSHMLock opens an existing shared-memory segment holding a given number of // POSIX semaphores. numLocks must match the number of locks the shared memory -// segment was created with and be a multiple of the lock bitmap size (default -// 32). -func OpenSHMLock(numLocks uint32) (*SHMLocks, error) { - if numLocks%bitmapSize != 0 || numLocks == 0 { - return nil, errors.Wrapf(syscall.EINVAL, "number of locks must be a multiple of %d", C.bitmap_size_c) +// segment was created with. +func OpenSHMLock(path string, numLocks uint32) (*SHMLocks, error) { + if numLocks == 0 { + return nil, errors.Wrapf(syscall.EINVAL, "number of locks must greater than 0") } locks := new(SHMLocks) + cPath := C.CString(path) + defer C.free(unsafe.Pointer(cPath)) + var errCode C.int - lockStruct := C.open_lock_shm(C.uint32_t(numLocks), &errCode) + lockStruct := C.open_lock_shm(cPath, C.uint32_t(numLocks), &errCode) if lockStruct == nil { // We got a null pointer, so something errored return nil, syscall.Errno(-1 * errCode) @@ -108,6 +117,8 @@ func (locks *SHMLocks) AllocateSemaphore() (uint32, error) { return 0, errors.Wrapf(syscall.EINVAL, "locks have already been closed") } + // This returns a U64, so we have the full u32 range available for + // semaphore indexes, and can still return error codes. retCode := C.allocate_semaphore(locks.lockStruct) if retCode < 0 { // Negative errno returned @@ -154,6 +165,10 @@ func (locks *SHMLocks) LockSemaphore(sem uint32) error { return errors.Wrapf(syscall.EINVAL, "given semaphore %d is higher than maximum locks count %d", sem, locks.maxLocks) } + // For pthread mutexes, we have to guarantee lock and unlock happen in + // the same thread. + runtime.LockOSThread() + retCode := C.lock_semaphore(locks.lockStruct, C.uint32_t(sem)) if retCode < 0 { // Negative errno returned @@ -184,5 +199,12 @@ func (locks *SHMLocks) UnlockSemaphore(sem uint32) error { return syscall.Errno(-1 * retCode) } + // For pthread mutexes, we have to guarantee lock and unlock happen in + // the same thread. + // OK if we take multiple locks - UnlockOSThread() won't actually unlock + // until the number of calls equals the number of calls to + // LockOSThread() + runtime.UnlockOSThread() + return nil } diff --git a/libpod/lock/shm/shm_lock.h b/libpod/lock/shm/shm_lock.h index 18bea47e9..8e7e23fb7 100644 --- a/libpod/lock/shm/shm_lock.h +++ b/libpod/lock/shm/shm_lock.h @@ -1,14 +1,11 @@ #ifndef shm_locks_h_ #define shm_locks_h_ -#include +#include #include // Magic number to ensure we open the right SHM segment -#define MAGIC 0xA5A5 - -// Name of the SHM -#define SHM_NAME "/libpod_lock" +#define MAGIC 0x87D1 // Type for our bitmaps typedef uint32_t bitmap_t; @@ -18,22 +15,28 @@ typedef uint32_t bitmap_t; // Struct to hold a single bitmap and associated locks typedef struct lock_group { - bitmap_t bitmap; - sem_t locks[BITMAP_SIZE]; + bitmap_t bitmap; + pthread_mutex_t locks[BITMAP_SIZE]; } lock_group_t; -// Struct to hold our SHM locks +// Struct to hold our SHM locks. +// Unused is required to be 0 in the current implementation. If we ever make +// changes to this structure in the future, this will be repurposed as a version +// field. typedef struct shm_struct { - uint16_t magic; - sem_t segment_lock; - uint32_t num_bitmaps; - uint32_t num_locks; - lock_group_t locks[]; + uint16_t magic; + uint16_t unused; + pthread_mutex_t segment_lock; + uint32_t num_bitmaps; + uint32_t num_locks; + lock_group_t locks[]; } shm_struct_t; -size_t compute_shm_size(uint32_t num_bitmaps); -shm_struct_t *setup_lock_shm(uint32_t num_locks, int *error_code); -shm_struct_t *open_lock_shm(uint32_t num_locks, int *error_code); +static size_t compute_shm_size(uint32_t num_bitmaps); +static int take_mutex(pthread_mutex_t *mutex); +static int release_mutex(pthread_mutex_t *mutex); +shm_struct_t *setup_lock_shm(char *path, uint32_t num_locks, int *error_code); +shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code); int32_t close_lock_shm(shm_struct_t *shm); int64_t allocate_semaphore(shm_struct_t *shm); int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index); diff --git a/libpod/lock/shm/shm_lock_test.go b/libpod/lock/shm/shm_lock_test.go index bc22db835..7174253d0 100644 --- a/libpod/lock/shm/shm_lock_test.go +++ b/libpod/lock/shm/shm_lock_test.go @@ -3,6 +3,7 @@ package shm import ( "fmt" "os" + "runtime" "syscall" "testing" "time" @@ -17,11 +18,13 @@ import ( // We can at least verify that the locks work within the local process. // 4 * BITMAP_SIZE to ensure we have to traverse bitmaps -const numLocks = 128 +const numLocks uint32 = 4 * BitmapSize + +const lockPath = "/libpod_test" // We need a test main to ensure that the SHM is created before the tests run func TestMain(m *testing.M) { - shmLock, err := CreateSHMLock(numLocks) + shmLock, err := CreateSHMLock(lockPath, numLocks) if err != nil { fmt.Fprintf(os.Stderr, "Error creating SHM for tests: %v\n", err) os.Exit(-1) @@ -42,19 +45,15 @@ func TestMain(m *testing.M) { } func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { - locks, err := OpenSHMLock(numLocks) + locks, err := OpenSHMLock(lockPath, numLocks) if err != nil { t.Fatalf("Error opening locks: %v", err) } defer func() { - // Unlock and deallocate all locks - // Ignore EBUSY (lock is already unlocked) + // Deallocate all locks // Ignore ENOENT (lock is not allocated) var i uint32 for i = 0; i < numLocks; i++ { - if err := locks.UnlockSemaphore(i); err != nil && err != syscall.EBUSY { - t.Fatalf("Error unlocking semaphore %d: %v", i, err) - } if err := locks.DeallocateSemaphore(i); err != nil && err != syscall.ENOENT { t.Fatalf("Error deallocating semaphore %d: %v", i, err) } @@ -73,16 +72,22 @@ func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { } } -// Test that creating an SHM with a bad size fails -func TestCreateNewSHMBadSize(t *testing.T) { +// Test that creating an SHM with a bad size rounds up to a good size +func TestCreateNewSHMBadSizeRoundsUp(t *testing.T) { // Odd number, not a power of 2, should never be a word size on a system - _, err := CreateSHMLock(7) - assert.Error(t, err) + lock, err := CreateSHMLock("/test1", 7) + assert.NoError(t, err) + + assert.Equal(t, lock.GetMaxLocks(), BitmapSize) + + if err := lock.Close(); err != nil { + t.Fatalf("Error closing locks: %v", err) + } } // Test that creating an SHM with 0 size fails func TestCreateNewSHMZeroSize(t *testing.T) { - _, err := CreateSHMLock(0) + _, err := CreateSHMLock("/test2", 0) assert.Error(t, err) } @@ -241,3 +246,28 @@ func TestLockSemaphoreActuallyLocks(t *testing.T) { assert.True(t, duration.Seconds() > 1.0) }) } + +// Test that locking and unlocking two semaphores succeeds +// Ensures that runtime.LockOSThread() is doing its job +func TestLockAndUnlockTwoSemaphore(t *testing.T) { + runLockTest(t, func(t *testing.T, locks *SHMLocks) { + err := locks.LockSemaphore(0) + assert.NoError(t, err) + + err = locks.LockSemaphore(1) + assert.NoError(t, err) + + err = locks.UnlockSemaphore(1) + assert.NoError(t, err) + + // Now yield scheduling + // To try and get us on another OS thread + runtime.Gosched() + + // And unlock the last semaphore + // If we are in a different OS thread, this should fail. + // However, runtime.UnlockOSThread() should guarantee we are not + err = locks.UnlockSemaphore(0) + assert.NoError(t, err) + }) +} diff --git a/libpod/lock/shm_lock_manager_linux.go b/libpod/lock/shm_lock_manager_linux.go index b1e9df12d..974431a13 100644 --- a/libpod/lock/shm_lock_manager_linux.go +++ b/libpod/lock/shm_lock_manager_linux.go @@ -3,13 +3,7 @@ package lock import ( - "fmt" - "math" - "strconv" - "syscall" - - "github.com/pkg/errors" - "github.com/projectatomic/libpod/libpod/lock/shm" + "github.com/containers/libpod/libpod/lock/shm" ) // SHMLockManager manages shared memory locks. @@ -18,8 +12,8 @@ type SHMLockManager struct { } // NewSHMLockManager makes a new SHMLockManager with the given number of locks. -func NewSHMLockManager(numLocks uint32) (Manager, error) { - locks, err := shm.CreateSHMLock(numLocks) +func NewSHMLockManager(path string, numLocks uint32) (Manager, error) { + locks, err := shm.CreateSHMLock(path, numLocks) if err != nil { return nil, err } @@ -32,8 +26,8 @@ func NewSHMLockManager(numLocks uint32) (Manager, error) { // OpenSHMLockManager opens an existing SHMLockManager with the given number of // locks. -func OpenSHMLockManager(numLocks uint32) (Manager, error) { - locks, err := shm.OpenSHMLock(numLocks) +func OpenSHMLockManager(path string, numLocks uint32) (Manager, error) { + locks, err := shm.OpenSHMLock(path, numLocks) if err != nil { return nil, err } @@ -59,27 +53,9 @@ func (m *SHMLockManager) AllocateLock() (Locker, error) { } // RetrieveLock retrieves a lock from the manager given its ID. -func (m *SHMLockManager) RetrieveLock(id string) (Locker, error) { - intID, err := strconv.ParseInt(id, 16, 64) - if err != nil { - return nil, errors.Wrapf(err, "given ID %q is not a valid SHMLockManager ID - cannot be parsed as int", id) - } - - if intID < 0 { - return nil, errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - must be positive", id) - } - - if intID > math.MaxUint32 { - return nil, errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - too large", id) - } - - var u32ID uint32 = uint32(intID) - if u32ID >= m.locks.GetMaxLocks() { - return nil, errors.Wrapf(syscall.EINVAL, "given ID %q is not a valid SHMLockManager ID - too large to fit", id) - } - +func (m *SHMLockManager) RetrieveLock(id uint32) (Locker, error) { lock := new(SHMLock) - lock.lockID = u32ID + lock.lockID = id lock.manager = m return lock, nil @@ -92,8 +68,8 @@ type SHMLock struct { } // ID returns the ID of the lock. -func (l *SHMLock) ID() string { - return fmt.Sprintf("%x", l.lockID) +func (l *SHMLock) ID() uint32 { + return l.lockID } // Lock acquires the lock. -- cgit v1.2.3-54-g00ecf From a364b656eaef1be5329abfd02d3fcd2dbcd37d64 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Thu, 23 Aug 2018 13:48:07 -0400 Subject: Add lock manager to libpod runtime Signed-off-by: Matthew Heon --- libpod/lock/lock.go | 11 ++-- libpod/lock/shm/shm_lock.go | 8 ++- libpod/lock/shm_lock_manager_linux.go | 12 ++-- libpod/runtime.go | 110 ++++++++++++++++++++++------------ 4 files changed, 95 insertions(+), 46 deletions(-) (limited to 'libpod/lock/lock.go') diff --git a/libpod/lock/lock.go b/libpod/lock/lock.go index 5258c641f..73c1fdcf7 100644 --- a/libpod/lock/lock.go +++ b/libpod/lock/lock.go @@ -41,11 +41,14 @@ type Locker interface { // Lock locks the lock. // This call MUST block until it successfully acquires the lock or // encounters a fatal error. - Lock() error + // All errors must be handled internally, as they are not returned. For + // the most part, panicking should be appropriate. + Lock() // Unlock unlocks the lock. - // A call to Unlock() on a lock that is already unlocked lock MUST - // error. - Unlock() error + // All errors must be handled internally, as they are not returned. For + // the most part, panicking should be appropriate. + // This includes unlocking locks which are already unlocked. + Unlock() // Free deallocates the underlying lock, allowing its reuse by other // pods and containers. // The lock MUST still be usable after a Free() - some libpod instances diff --git a/libpod/lock/shm/shm_lock.go b/libpod/lock/shm/shm_lock.go index 16d7f2008..3372a8c71 100644 --- a/libpod/lock/shm/shm_lock.go +++ b/libpod/lock/shm/shm_lock.go @@ -12,9 +12,13 @@ import ( "unsafe" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) -const ( +var ( + // BitmapSize is the size of the bitmap used when managing SHM locks. + // an SHM lock manager's max locks will be rounded up to a multiple of + // this number. BitmapSize uint32 = uint32(C.bitmap_size_c) ) @@ -51,6 +55,8 @@ func CreateSHMLock(path string, numLocks uint32) (*SHMLocks, error) { locks.maxLocks = uint32(lockStruct.num_locks) locks.valid = true + logrus.Debugf("Initialized SHM lock manager at path %s", path) + return locks, nil } diff --git a/libpod/lock/shm_lock_manager_linux.go b/libpod/lock/shm_lock_manager_linux.go index 974431a13..2c0ea611a 100644 --- a/libpod/lock/shm_lock_manager_linux.go +++ b/libpod/lock/shm_lock_manager_linux.go @@ -73,13 +73,17 @@ func (l *SHMLock) ID() uint32 { } // Lock acquires the lock. -func (l *SHMLock) Lock() error { - return l.manager.locks.LockSemaphore(l.lockID) +func (l *SHMLock) Lock() { + if err := l.manager.locks.LockSemaphore(l.lockID); err != nil { + panic(err.Error()) + } } // Unlock releases the lock. -func (l *SHMLock) Unlock() error { - return l.manager.locks.UnlockSemaphore(l.lockID) +func (l *SHMLock) Unlock() { + if err := l.manager.locks.UnlockSemaphore(l.lockID); err != nil { + panic(err.Error()) + } } // Free releases the lock, allowing it to be reused. diff --git a/libpod/runtime.go b/libpod/runtime.go index facbe5d66..238a7a9db 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -11,6 +11,7 @@ import ( is "github.com/containers/image/storage" "github.com/containers/image/types" "github.com/containers/libpod/libpod/image" + "github.com/containers/libpod/libpod/lock" "github.com/containers/libpod/pkg/firewall" sysreg "github.com/containers/libpod/pkg/registries" "github.com/containers/libpod/pkg/rootless" @@ -64,6 +65,11 @@ const ( // DefaultInitPath is the default path to the container-init binary DefaultInitPath = "/usr/libexec/podman/catatonit" + + // DefaultSHMLockPath is the default path for SHM locks + DefaultSHMLockPath = "/libpod_lock" + // DefaultRootlessSHMLockPath is the default path for rootless SHM locks + DefaultRootlessSHMLockPath = "/libpod_rootless_lock" ) // A RuntimeOption is a functional option which alters the Runtime created by @@ -86,6 +92,7 @@ type Runtime struct { lock sync.RWMutex imageRuntime *image.Runtime firewallBackend firewall.FirewallBackend + lockManager lock.Manager configuredFrom *runtimeConfiguredFrom } @@ -165,6 +172,7 @@ type RuntimeConfig struct { // and all containers and pods will be visible. // The default namespace is "". Namespace string `toml:"namespace,omitempty"` + // InfraImage is the image a pod infra container will use to manage namespaces InfraImage string `toml:"infra_image"` // InfraCommand is the command run to start up a pod infra container @@ -179,6 +187,10 @@ type RuntimeConfig struct { EnablePortReservation bool `toml:"enable_port_reservation"` // EnableLabeling indicates wether libpod will support container labeling EnableLabeling bool `toml:"label"` + + // NumLocks is the number of locks to make available for containers and + // pods. + NumLocks uint32 `toml:"num_locks,omitempty"` } // runtimeConfiguredFrom is a struct used during early runtime init to help @@ -234,6 +246,7 @@ var ( InfraImage: DefaultInfraImage, EnablePortReservation: true, EnableLabeling: true, + NumLocks: 2048, } ) @@ -487,6 +500,56 @@ func makeRuntime(runtime *Runtime) (err error) { } } + // We now need to see if the system has restarted + // We check for the presence of a file in our tmp directory to verify this + // This check must be locked to prevent races + runtimeAliveLock := filepath.Join(runtime.config.TmpDir, "alive.lck") + runtimeAliveFile := filepath.Join(runtime.config.TmpDir, "alive") + aliveLock, err := storage.GetLockfile(runtimeAliveLock) + if err != nil { + return errors.Wrapf(err, "error acquiring runtime init lock") + } + // Acquire the lock and hold it until we return + // This ensures that no two processes will be in runtime.refresh at once + // TODO: we can't close the FD in this lock, so we should keep it around + // and use it to lock important operations + aliveLock.Lock() + locked := true + doRefresh := false + defer func() { + if locked { + aliveLock.Unlock() + } + }() + _, err = os.Stat(runtimeAliveFile) + if err != nil { + // If the file doesn't exist, we need to refresh the state + // This will trigger on first use as well, but refreshing an + // empty state only creates a single file + // As such, it's not really a performance concern + if os.IsNotExist(err) { + doRefresh = true + } else { + return errors.Wrapf(err, "error reading runtime status file %s", runtimeAliveFile) + } + } + + // Set up the lock manager + var manager lock.Manager + lockPath := DefaultSHMLockPath + if rootless.IsRootless() { + lockPath = DefaultRootlessSHMLockPath + } + if doRefresh { + manager, err = lock.NewSHMLockManager(lockPath, runtime.config.NumLocks) + } else { + manager, err = lock.OpenSHMLockManager(lockPath, runtime.config.NumLocks) + } + if err != nil { + return errors.Wrapf(err, "error initializing SHM locking") + } + runtime.lockManager = manager + // Set up the state switch runtime.config.StateType { case InMemoryStateStore: @@ -656,46 +719,19 @@ func makeRuntime(runtime *Runtime) (err error) { } runtime.firewallBackend = fwBackend - // We now need to see if the system has restarted - // We check for the presence of a file in our tmp directory to verify this - // This check must be locked to prevent races - runtimeAliveLock := filepath.Join(runtime.config.TmpDir, "alive.lck") - runtimeAliveFile := filepath.Join(runtime.config.TmpDir, "alive") - aliveLock, err := storage.GetLockfile(runtimeAliveLock) - if err != nil { - return errors.Wrapf(err, "error acquiring runtime init lock") - } - // Acquire the lock and hold it until we return - // This ensures that no two processes will be in runtime.refresh at once - // TODO: we can't close the FD in this lock, so we should keep it around - // and use it to lock important operations - aliveLock.Lock() - locked := true - defer func() { - if locked { + // If we need to refresh the state, do it now - things are guaranteed to + // be set up by now. + if doRefresh { + if os.Geteuid() != 0 { aliveLock.Unlock() - } - }() - _, err = os.Stat(runtimeAliveFile) - if err != nil { - // If the file doesn't exist, we need to refresh the state - // This will trigger on first use as well, but refreshing an - // empty state only creates a single file - // As such, it's not really a performance concern - if os.IsNotExist(err) { - if os.Geteuid() != 0 { - aliveLock.Unlock() - locked = false - if err2 := runtime.refreshRootless(); err2 != nil { - return err2 - } - } else { - if err2 := runtime.refresh(runtimeAliveFile); err2 != nil { - return err2 - } + locked = false + if err2 := runtime.refreshRootless(); err2 != nil { + return err2 } } else { - return errors.Wrapf(err, "error reading runtime status file %s", runtimeAliveFile) + if err2 := runtime.refresh(runtimeAliveFile); err2 != nil { + return err2 + } } } -- cgit v1.2.3-54-g00ecf