From 7fdd20ae5a1ced1faceab9cb0a6e553343911a0b Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Thu, 14 Feb 2019 17:25:58 -0500 Subject: Add initial version of renumber backend Renumber is a way of renumbering container locks after the number of locks available has changed. For now, renumber only works with containers. Signed-off-by: Matthew Heon --- libpod/lock/shm/shm_lock.c | 30 ++++++++++++++++++++++++++++++ libpod/lock/shm/shm_lock.go | 16 ++++++++++++++++ libpod/lock/shm/shm_lock.h | 1 + libpod/lock/shm/shm_lock_test.go | 28 +++++++++++++++++++++------- 4 files changed, 68 insertions(+), 7 deletions(-) (limited to 'libpod/lock/shm') diff --git a/libpod/lock/shm/shm_lock.c b/libpod/lock/shm/shm_lock.c index 4af58d857..367055823 100644 --- a/libpod/lock/shm/shm_lock.c +++ b/libpod/lock/shm/shm_lock.c @@ -407,6 +407,36 @@ int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index) { return 0; } +// Deallocate all semaphores unconditionally. +// Returns negative ERRNO values. +int32_t deallocate_all_semaphores(shm_struct_t *shm) { + int ret_code; + uint i; + + if (shm == NULL) { + return -1 * EINVAL; + } + + // Lock the mutex controlling access to our shared memory + ret_code = take_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } + + // Iterate through all bitmaps and reset to unused + for (i = 0; i < shm->num_bitmaps; i++) { + shm->locks[i].bitmap = 0; + } + + // Unlock the allocation control mutex + ret_code = release_mutex(&(shm->segment_lock)); + if (ret_code != 0) { + return -1 * ret_code; + } + + return 0; +} + // Lock a given semaphore // Does not check if the semaphore is allocated - this ensures that, even for // removed containers, we can still successfully lock to check status (and diff --git a/libpod/lock/shm/shm_lock.go b/libpod/lock/shm/shm_lock.go index 87d28e5c1..e70ea8743 100644 --- a/libpod/lock/shm/shm_lock.go +++ b/libpod/lock/shm/shm_lock.go @@ -155,6 +155,22 @@ func (locks *SHMLocks) DeallocateSemaphore(sem uint32) error { return nil } +// DeallocateAllSemaphores frees all semaphores so they can be reallocated to +// other containers and pods. +func (locks *SHMLocks) DeallocateAllSemaphores() error { + if !locks.valid { + return errors.Wrapf(syscall.EINVAL, "locks have already been closed") + } + + retCode := C.deallocate_all_semaphores(locks.lockStruct) + if retCode < 0 { + // Negative errno return from C + return syscall.Errno(-1 * retCode) + } + + return nil +} + // LockSemaphore locks the given semaphore. // If the semaphore is already locked, LockSemaphore will block until the lock // can be acquired. diff --git a/libpod/lock/shm/shm_lock.h b/libpod/lock/shm/shm_lock.h index 8e7e23fb7..58e4297e2 100644 --- a/libpod/lock/shm/shm_lock.h +++ b/libpod/lock/shm/shm_lock.h @@ -40,6 +40,7 @@ shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code); int32_t close_lock_shm(shm_struct_t *shm); int64_t allocate_semaphore(shm_struct_t *shm); int32_t deallocate_semaphore(shm_struct_t *shm, uint32_t sem_index); +int32_t deallocate_all_semaphores(shm_struct_t *shm); int32_t lock_semaphore(shm_struct_t *shm, uint32_t sem_index); int32_t unlock_semaphore(shm_struct_t *shm, uint32_t sem_index); diff --git a/libpod/lock/shm/shm_lock_test.go b/libpod/lock/shm/shm_lock_test.go index 594eb5d8e..830035881 100644 --- a/libpod/lock/shm/shm_lock_test.go +++ b/libpod/lock/shm/shm_lock_test.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "runtime" - "syscall" "testing" "time" @@ -53,12 +52,8 @@ func runLockTest(t *testing.T, testFunc func(*testing.T, *SHMLocks)) { } defer func() { // Deallocate all locks - // Ignore ENOENT (lock is not allocated) - var i uint32 - for i = 0; i < numLocks; i++ { - if err := locks.DeallocateSemaphore(i); err != nil && err != syscall.ENOENT { - t.Fatalf("Error deallocating semaphore %d: %v", i, err) - } + if err := locks.DeallocateAllSemaphores(); err != nil { + t.Fatalf("Error deallocating semaphores: %v", err) } if err := locks.Close(); err != nil { @@ -212,6 +207,25 @@ func TestAllocateDeallocateCycle(t *testing.T) { }) } +// Test that DeallocateAllSemaphores deallocates all semaphores +func TestDeallocateAllSemaphoresDeallocatesAll(t *testing.T) { + runLockTest(t, func(t *testing.T, locks *SHMLocks) { + // Allocate a lock + locks1, err := locks.AllocateSemaphore() + assert.NoError(t, err) + + // Free all locks + err = locks.DeallocateAllSemaphores() + assert.NoError(t, err) + + // Allocate another lock + locks2, err := locks.AllocateSemaphore() + assert.NoError(t, err) + + assert.Equal(t, locks1, locks2) + }) +} + // Test that locks actually lock func TestLockSemaphoreActuallyLocks(t *testing.T) { runLockTest(t, func(t *testing.T, locks *SHMLocks) { -- cgit v1.2.3-54-g00ecf From f9c548219b6543959dd240618f8a922fdbcabc6d Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Fri, 15 Feb 2019 10:33:59 -0500 Subject: Recreate SHM locks when renumbering on count mismatch When we're renumbering locks, we're destroying all existing allocations anyways, so destroying the old lock struct is not a particularly big deal. Existing long-lived libpod instances will continue to use the old locks, but that will be solved in a followon. Also, solve an issue with returning error values in the C code. There were a few places where we return ERRNO where it was not set, so make them return actual error codes). Signed-off-by: Matthew Heon --- libpod/lock/shm/shm_lock.c | 6 ++++-- libpod/runtime.go | 26 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) (limited to 'libpod/lock/shm') diff --git a/libpod/lock/shm/shm_lock.c b/libpod/lock/shm/shm_lock.c index 367055823..d11fce71a 100644 --- a/libpod/lock/shm/shm_lock.c +++ b/libpod/lock/shm/shm_lock.c @@ -203,6 +203,8 @@ shm_struct_t *setup_lock_shm(char *path, uint32_t num_locks, int *error_code) { // terminating NULL byte. // Returns a valid pointer on success or NULL on error. // If an error occurs, negative ERRNO values will be written to error_code. +// ERANGE is returned for a mismatch between num_locks and the number of locks +// available in the the SHM lock struct. shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code) { int shm_fd; shm_struct_t *shm; @@ -255,11 +257,11 @@ shm_struct_t *open_lock_shm(char *path, uint32_t num_locks, int *error_code) { // Need to check the SHM to see if it's actually our locks if (shm->magic != MAGIC) { - *error_code = -1 * errno; + *error_code = -1 * EBADF; goto CLEANUP; } if (shm->num_locks != (num_bitmaps * BITMAP_SIZE)) { - *error_code = -1 * errno; + *error_code = -1 * ERANGE; goto CLEANUP; } diff --git a/libpod/runtime.go b/libpod/runtime.go index bc7c061c4..850df4fc9 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -7,6 +7,7 @@ import ( "os/exec" "path/filepath" "sync" + "syscall" "github.com/BurntSushi/toml" is "github.com/containers/image/storage" @@ -757,12 +758,6 @@ func makeRuntime(runtime *Runtime) (err error) { aliveLock.Unlock() } }() - // If we're renumbering locks, do it now. - // It breaks out of normal runtime init, and will not return a valid - // runtime. - if runtime.doRenumber { - return runtime.renumberLocks() - } _, err = os.Stat(runtimeAliveFile) if err != nil { @@ -785,6 +780,18 @@ func makeRuntime(runtime *Runtime) (err error) { manager, err := lock.OpenSHMLockManager(lockPath, runtime.config.NumLocks) if err != nil { if os.IsNotExist(errors.Cause(err)) { + manager, err = lock.NewSHMLockManager(lockPath, runtime.config.NumLocks) + if err != nil { + return err + } + } else if err == syscall.ERANGE && runtime.doRenumber { + // ERANGE indicates a lock numbering mismatch. + // Since we're renumbering, this is not fatal. + // Remove the earlier set of locks and recreate. + if err := os.Remove(filepath.Join("/dev/shm", lockPath)); err != nil { + return errors.Wrapf(err, "error removing libpod locks file %s", lockPath) + } + manager, err = lock.NewSHMLockManager(lockPath, runtime.config.NumLocks) if err != nil { return err @@ -795,6 +802,13 @@ func makeRuntime(runtime *Runtime) (err error) { } runtime.lockManager = manager + // If we're renumbering locks, do it now. + // It breaks out of normal runtime init, and will not return a valid + // runtime. + if runtime.doRenumber { + return runtime.renumberLocks() + } + // If we need to refresh the state, do it now - things are guaranteed to // be set up by now. if doRefresh { -- cgit v1.2.3-54-g00ecf