Add basic deadlock detection for container start/remove

We can easily tell if we're going to deadlock by comparing lock
IDs before actually taking the lock. Add a few checks for this in
common places where deadlocks might occur.

This does not yet cover pod operations, where detection is more
difficult (and costly) due to the number of locks being involved
being higher than 2.

Also, add some error wrapping on the Podman side, so we can tell
people to use `system renumber` when it occurs.

Signed-off-by: Matthew Heon <matthew.heon@pm.me>

1 files changed, 13 insertions, 0 deletions

diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 78057e3f9..08e19edb8 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -469,6 +469,10 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode
 				logrus.Debugf("unable to remove container %s after failing to start and attach to it", ctr.ID())
 			}
 		}
+		if errors.Cause(err) == define.ErrWillDeadlock {
+			logrus.Debugf("Deadlock error: %v", err)
+			return define.ExitCode(err), errors.Errorf("attempting to start container %s would cause a deadlock; please run 'podman system renumber' to resolve", ctr.ID())
+		}
 		return define.ExitCode(err), err
 	}
 
@@ -702,6 +706,11 @@ func (r *LocalRuntime) Start(ctx context.Context, c *cliconfig.StartValues, sigP
 				return exitCode, nil
 			}
 
+			if errors.Cause(err) == define.ErrWillDeadlock {
+				logrus.Debugf("Deadlock error: %v", err)
+				return define.ExitCode(err), errors.Errorf("attempting to start container %s would cause a deadlock; please run 'podman system renumber' to resolve", ctr.ID())
+			}
+
 			if ctrRunning {
 				return 0, err
 			}
@@ -735,6 +744,10 @@ func (r *LocalRuntime) Start(ctx context.Context, c *cliconfig.StartValues, sigP
 				if lastError != nil {
 					fmt.Fprintln(os.Stderr, lastError)
 				}
+				if errors.Cause(err) == define.ErrWillDeadlock {
+					lastError = errors.Wrapf(err, "please run 'podman system renumber' to resolve deadlocks")
+					continue
+				}
 				lastError = errors.Wrapf(err, "unable to start container %q", container)
 				continue
 			}