Initial checkin from CRI-O repo

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

3 files changed, 744 insertions, 0 deletions

diff --git a/cmd/crio/config.go b/cmd/crio/config.go
new file mode 100644
index 000000000..76e3361d4
--- /dev/null
+++ b/cmd/crio/config.go
@@ -0,0 +1,192 @@
+package main
+
+import (
+	"os"
+	"text/template"
+
+	"github.com/kubernetes-incubator/cri-o/server"
+	"github.com/urfave/cli"
+)
+
+var commentedConfigTemplate = template.Must(template.New("config").Parse(`
+# The "crio" table contains all of the server options.
+[crio]
+
+# root is a path to the "root directory". CRIO stores all of its data,
+# including container images, in this directory.
+root = "{{ .Root }}"
+
+# run is a path to the "run directory". CRIO stores all of its state
+# in this directory.
+runroot = "{{ .RunRoot }}"
+
+# storage_driver select which storage driver is used to manage storage
+# of images and containers.
+storage_driver = "{{ .Storage }}"
+
+# storage_option is used to pass an option to the storage driver.
+storage_option = [
+{{ range $opt := .StorageOptions }}{{ printf "\t%q,\n" $opt }}{{ end }}]
+
+# The "crio.api" table contains settings for the kubelet/gRPC
+# interface (which is also used by crioctl).
+[crio.api]
+
+# listen is the path to the AF_LOCAL socket on which crio will listen.
+listen = "{{ .Listen }}"
+
+# stream_address is the IP address on which the stream server will listen
+stream_address = "{{ .StreamAddress }}"
+
+# stream_port is the port on which the stream server will listen
+stream_port = "{{ .StreamPort }}"
+
+# file_locking is whether file-based locking will be used instead of
+# in-memory locking
+file_locking = {{ .FileLocking }}
+
+# The "crio.runtime" table contains settings pertaining to the OCI
+# runtime used and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "{{ .Runtime }}"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = "{{ .RuntimeUntrustedWorkload }}"
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "{{ .DefaultWorkloadTrust }}"
+
+# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
+no_pivot = {{ .NoPivot }}
+
+# conmon is the path to conmon binary, used for managing the runtime.
+conmon = "{{ .Conmon }}"
+
+# conmon_env is the environment variable list for conmon process,
+# used for passing necessary environment variable to conmon or runtime.
+conmon_env = [
+{{ range $env := .ConmonEnv }}{{ printf "\t%q,\n" $env }}{{ end }}]
+
+# selinux indicates whether or not SELinux will be used for pod
+# separation on the host. If you enable this flag, SELinux must be running
+# on the host.
+selinux = {{ .SELinux }}
+
+# seccomp_profile is the seccomp json profile path which is used as the
+# default for the runtime.
+seccomp_profile = "{{ .SeccompProfile }}"
+
+# apparmor_profile is the apparmor profile name which is used as the
+# default for the runtime.
+apparmor_profile = "{{ .ApparmorProfile }}"
+
+# cgroup_manager is the cgroup management implementation to be used
+# for the runtime.
+cgroup_manager = "{{ .CgroupManager }}"
+
+# hooks_dir_path is the oci hooks directory for automatically executed hooks
+hooks_dir_path = "{{ .HooksDirPath }}"
+
+# default_mounts is the mounts list to be mounted for the container when created
+default_mounts = [
+{{ range $mount := .DefaultMounts }}{{ printf "\t%q, \n" $mount }}{{ end }}]
+
+# pids_limit is the number of processes allowed in a container
+pids_limit = {{ .PidsLimit }}
+
+# log_size_max is the max limit for the container log size in bytes.
+# Negative values indicate that no limit is imposed.
+log_size_max = {{ .LogSizeMax }}
+
+# The "crio.image" table contains settings pertaining to the
+# management of OCI images.
+[crio.image]
+
+# default_transport is the prefix we try prepending to an image name if the
+# image name as we receive it can't be parsed as a valid source reference
+default_transport = "{{ .DefaultTransport }}"
+
+# pause_image is the image which we use to instantiate infra containers.
+pause_image = "{{ .PauseImage }}"
+
+# pause_command is the command to run in a pause_image to have a container just
+# sit there.  If the image contains the necessary information, this value need
+# not be specified.
+pause_command = "{{ .PauseCommand }}"
+
+# signature_policy is the name of the file which decides what sort of policy we
+# use when deciding whether or not to trust an image that we've pulled.
+# Outside of testing situations, it is strongly advised that this be left
+# unspecified so that the default system-wide policy will be used.
+signature_policy = "{{ .SignaturePolicyPath }}"
+
+# image_volumes controls how image volumes are handled.
+# The valid values are mkdir and ignore.
+image_volumes = "{{ .ImageVolumes }}"
+
+# insecure_registries is used to skip TLS verification when pulling images.
+insecure_registries = [
+{{ range $opt := .InsecureRegistries }}{{ printf "\t%q,\n" $opt }}{{ end }}]
+
+# registries is used to specify a comma separated list of registries to be used
+# when pulling an unqualified image (e.g. fedora:rawhide).
+registries = [
+{{ range $opt := .Registries }}{{ printf "\t%q,\n" $opt }}{{ end }}]
+
+# The "crio.network" table contains settings pertaining to the
+# management of CNI plugins.
+[crio.network]
+
+# network_dir is is where CNI network configuration
+# files are stored.
+network_dir = "{{ .NetworkDir }}"
+
+# plugin_dir is is where CNI plugin binaries are stored.
+plugin_dir = "{{ .PluginDir }}"
+`))
+
+// TODO: Currently ImageDir isn't really used, so we haven't added it to this
+//       template. Add it once the storage code has been merged.
+
+var configCommand = cli.Command{
+	Name:  "config",
+	Usage: "generate crio configuration files",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "default",
+			Usage: "output the default configuration",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		// At this point, app.Before has already parsed the user's chosen
+		// config file. So no need to handle that here.
+		config := c.App.Metadata["config"].(*server.Config)
+		if c.Bool("default") {
+			config = server.DefaultConfig()
+		}
+
+		// Output the commented config.
+		return commentedConfigTemplate.ExecuteTemplate(os.Stdout, "config", config)
+	},
+}
diff --git a/cmd/crio/daemon_linux.go b/cmd/crio/daemon_linux.go
new file mode 100644
index 000000000..884d3f269
--- /dev/null
+++ b/cmd/crio/daemon_linux.go
@@ -0,0 +1,20 @@
+// +build linux
+
+package main
+
+import (
+	systemdDaemon "github.com/coreos/go-systemd/daemon"
+	"github.com/sirupsen/logrus"
+)
+
+func sdNotify() {
+	if _, err := systemdDaemon.SdNotify(true, "READY=1"); err != nil {
+		logrus.Warnf("Failed to sd_notify systemd: %v", err)
+	}
+}
+
+// notifySystem sends a message to the host when the server is ready to be used
+func notifySystem() {
+	// Tell the init daemon we are accepting requests
+	go sdNotify()
+}
diff --git a/cmd/crio/main.go b/cmd/crio/main.go
new file mode 100644
index 000000000..e58adb114
--- /dev/null
+++ b/cmd/crio/main.go
@@ -0,0 +1,532 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"os/signal"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/containers/storage/pkg/reexec"
+	"github.com/kubernetes-incubator/cri-o/libkpod"
+	"github.com/kubernetes-incubator/cri-o/server"
+	"github.com/kubernetes-incubator/cri-o/version"
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/sirupsen/logrus"
+	"github.com/soheilhy/cmux"
+	"github.com/urfave/cli"
+	"golang.org/x/sys/unix"
+	"google.golang.org/grpc"
+	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+)
+
+// gitCommit is the commit that the binary is being built from.
+// It will be populated by the Makefile.
+var gitCommit = ""
+
+func validateConfig(config *server.Config) error {
+	switch config.ImageVolumes {
+	case libkpod.ImageVolumesMkdir:
+	case libkpod.ImageVolumesIgnore:
+	case libkpod.ImageVolumesBind:
+	default:
+		return fmt.Errorf("Unrecognized image volume type specified")
+
+	}
+
+	// This needs to match the read buffer size in conmon
+	if config.LogSizeMax >= 0 && config.LogSizeMax < 8192 {
+		return fmt.Errorf("log size max should be negative or >= 8192")
+	}
+	return nil
+}
+
+func mergeConfig(config *server.Config, ctx *cli.Context) error {
+	// Don't parse the config if the user explicitly set it to "".
+	if path := ctx.GlobalString("config"); path != "" {
+		if err := config.UpdateFromFile(path); err != nil {
+			if ctx.GlobalIsSet("config") || !os.IsNotExist(err) {
+				return err
+			}
+
+			// We don't error out if --config wasn't explicitly set and the
+			// default doesn't exist. But we will log a warning about it, so
+			// the user doesn't miss it.
+			logrus.Warnf("default configuration file does not exist: %s", server.CrioConfigPath)
+		}
+	}
+
+	// Override options set with the CLI.
+	if ctx.GlobalIsSet("conmon") {
+		config.Conmon = ctx.GlobalString("conmon")
+	}
+	if ctx.GlobalIsSet("pause-command") {
+		config.PauseCommand = ctx.GlobalString("pause-command")
+	}
+	if ctx.GlobalIsSet("pause-image") {
+		config.PauseImage = ctx.GlobalString("pause-image")
+	}
+	if ctx.GlobalIsSet("signature-policy") {
+		config.SignaturePolicyPath = ctx.GlobalString("signature-policy")
+	}
+	if ctx.GlobalIsSet("root") {
+		config.Root = ctx.GlobalString("root")
+	}
+	if ctx.GlobalIsSet("runroot") {
+		config.RunRoot = ctx.GlobalString("runroot")
+	}
+	if ctx.GlobalIsSet("storage-driver") {
+		config.Storage = ctx.GlobalString("storage-driver")
+	}
+	if ctx.GlobalIsSet("storage-opt") {
+		config.StorageOptions = ctx.GlobalStringSlice("storage-opt")
+	}
+	if ctx.GlobalIsSet("file-locking") {
+		config.FileLocking = ctx.GlobalBool("file-locking")
+	}
+	if ctx.GlobalIsSet("insecure-registry") {
+		config.InsecureRegistries = ctx.GlobalStringSlice("insecure-registry")
+	}
+	if ctx.GlobalIsSet("registry") {
+		config.Registries = ctx.GlobalStringSlice("registry")
+	}
+	if ctx.GlobalIsSet("default-transport") {
+		config.DefaultTransport = ctx.GlobalString("default-transport")
+	}
+	if ctx.GlobalIsSet("listen") {
+		config.Listen = ctx.GlobalString("listen")
+	}
+	if ctx.GlobalIsSet("stream-address") {
+		config.StreamAddress = ctx.GlobalString("stream-address")
+	}
+	if ctx.GlobalIsSet("stream-port") {
+		config.StreamPort = ctx.GlobalString("stream-port")
+	}
+	if ctx.GlobalIsSet("runtime") {
+		config.Runtime = ctx.GlobalString("runtime")
+	}
+	if ctx.GlobalIsSet("selinux") {
+		config.SELinux = ctx.GlobalBool("selinux")
+	}
+	if ctx.GlobalIsSet("seccomp-profile") {
+		config.SeccompProfile = ctx.GlobalString("seccomp-profile")
+	}
+	if ctx.GlobalIsSet("apparmor-profile") {
+		config.ApparmorProfile = ctx.GlobalString("apparmor-profile")
+	}
+	if ctx.GlobalIsSet("cgroup-manager") {
+		config.CgroupManager = ctx.GlobalString("cgroup-manager")
+	}
+	if ctx.GlobalIsSet("hooks-dir-path") {
+		config.HooksDirPath = ctx.GlobalString("hooks-dir-path")
+	}
+	if ctx.GlobalIsSet("default-mounts") {
+		config.DefaultMounts = ctx.GlobalStringSlice("default-mounts")
+	}
+	if ctx.GlobalIsSet("pids-limit") {
+		config.PidsLimit = ctx.GlobalInt64("pids-limit")
+	}
+	if ctx.GlobalIsSet("log-size-max") {
+		config.LogSizeMax = ctx.GlobalInt64("log-size-max")
+	}
+	if ctx.GlobalIsSet("cni-config-dir") {
+		config.NetworkDir = ctx.GlobalString("cni-config-dir")
+	}
+	if ctx.GlobalIsSet("cni-plugin-dir") {
+		config.PluginDir = ctx.GlobalString("cni-plugin-dir")
+	}
+	if ctx.GlobalIsSet("image-volumes") {
+		config.ImageVolumes = libkpod.ImageVolumesType(ctx.GlobalString("image-volumes"))
+	}
+	return nil
+}
+
+func catchShutdown(gserver *grpc.Server, sserver *server.Server, hserver *http.Server, signalled *bool) {
+	sig := make(chan os.Signal, 10)
+	signal.Notify(sig, unix.SIGINT, unix.SIGTERM)
+	go func() {
+		for s := range sig {
+			switch s {
+			case unix.SIGINT:
+				logrus.Debugf("Caught SIGINT")
+			case unix.SIGTERM:
+				logrus.Debugf("Caught SIGTERM")
+			default:
+				continue
+			}
+			*signalled = true
+			gserver.GracefulStop()
+			hserver.Shutdown(context.Background())
+			// TODO(runcom): enable this after https://github.com/kubernetes/kubernetes/pull/51377
+			//sserver.StopStreamServer()
+			sserver.StopExitMonitor()
+			if err := sserver.Shutdown(); err != nil {
+				logrus.Warnf("error shutting down main service %v", err)
+			}
+			return
+		}
+	}()
+}
+
+func main() {
+	if reexec.Init() {
+		return
+	}
+	app := cli.NewApp()
+
+	var v []string
+	v = append(v, version.Version)
+	if gitCommit != "" {
+		v = append(v, fmt.Sprintf("commit: %s", gitCommit))
+	}
+	app.Name = "crio"
+	app.Usage = "crio server"
+	app.Version = strings.Join(v, "\n")
+	app.Metadata = map[string]interface{}{
+		"config": server.DefaultConfig(),
+	}
+
+	app.Flags = []cli.Flag{
+		cli.StringFlag{
+			Name:  "config",
+			Value: server.CrioConfigPath,
+			Usage: "path to configuration file",
+		},
+		cli.StringFlag{
+			Name:  "conmon",
+			Usage: "path to the conmon executable",
+		},
+		cli.StringFlag{
+			Name:  "listen",
+			Usage: "path to crio socket",
+		},
+		cli.StringFlag{
+			Name:  "stream-address",
+			Usage: "bind address for streaming socket",
+		},
+		cli.StringFlag{
+			Name:  "stream-port",
+			Usage: "bind port for streaming socket (default: \"10010\")",
+		},
+		cli.StringFlag{
+			Name:  "log",
+			Value: "",
+			Usage: "set the log file path where internal debug information is written",
+		},
+		cli.StringFlag{
+			Name:  "log-format",
+			Value: "text",
+			Usage: "set the format used by logs ('text' (default), or 'json')",
+		},
+		cli.StringFlag{
+			Name:  "log-level",
+			Usage: "log messages above specified level: debug, info (default), warn, error, fatal or panic",
+		},
+
+		cli.StringFlag{
+			Name:  "pause-command",
+			Usage: "name of the pause command in the pause image",
+		},
+		cli.StringFlag{
+			Name:  "pause-image",
+			Usage: "name of the pause image",
+		},
+		cli.StringFlag{
+			Name:  "signature-policy",
+			Usage: "path to signature policy file",
+		},
+		cli.StringFlag{
+			Name:  "root",
+			Usage: "crio root dir",
+		},
+		cli.StringFlag{
+			Name:  "runroot",
+			Usage: "crio state dir",
+		},
+		cli.StringFlag{
+			Name:  "storage-driver",
+			Usage: "storage driver",
+		},
+		cli.StringSliceFlag{
+			Name:  "storage-opt",
+			Usage: "storage driver option",
+		},
+		cli.BoolFlag{
+			Name:  "file-locking",
+			Usage: "enable or disable file-based locking",
+		},
+		cli.StringSliceFlag{
+			Name:  "insecure-registry",
+			Usage: "whether to disable TLS verification for the given registry",
+		},
+		cli.StringSliceFlag{
+			Name:  "registry",
+			Usage: "registry to be prepended when pulling unqualified images, can be specified multiple times",
+		},
+		cli.StringFlag{
+			Name:  "default-transport",
+			Usage: "default transport",
+		},
+		cli.StringFlag{
+			Name:  "runtime",
+			Usage: "OCI runtime path",
+		},
+		cli.StringFlag{
+			Name:  "seccomp-profile",
+			Usage: "default seccomp profile path",
+		},
+		cli.StringFlag{
+			Name:  "apparmor-profile",
+			Usage: "default apparmor profile name (default: \"crio-default\")",
+		},
+		cli.BoolFlag{
+			Name:  "selinux",
+			Usage: "enable selinux support",
+		},
+		cli.StringFlag{
+			Name:  "cgroup-manager",
+			Usage: "cgroup manager (cgroupfs or systemd)",
+		},
+		cli.Int64Flag{
+			Name:  "pids-limit",
+			Value: libkpod.DefaultPidsLimit,
+			Usage: "maximum number of processes allowed in a container",
+		},
+		cli.Int64Flag{
+			Name:  "log-size-max",
+			Value: libkpod.DefaultLogSizeMax,
+			Usage: "maximum log size in bytes for a container",
+		},
+		cli.StringFlag{
+			Name:  "cni-config-dir",
+			Usage: "CNI configuration files directory",
+		},
+		cli.StringFlag{
+			Name:  "cni-plugin-dir",
+			Usage: "CNI plugin binaries directory",
+		},
+		cli.StringFlag{
+			Name:  "image-volumes",
+			Value: string(libkpod.ImageVolumesMkdir),
+			Usage: "image volume handling ('mkdir', 'bind', or 'ignore')",
+		},
+		cli.StringFlag{
+			Name:   "hooks-dir-path",
+			Usage:  "set the OCI hooks directory path",
+			Value:  libkpod.DefaultHooksDirPath,
+			Hidden: true,
+		},
+		cli.StringSliceFlag{
+			Name:   "default-mounts",
+			Usage:  "add one or more default mount paths in the form host:container",
+			Hidden: true,
+		},
+		cli.BoolFlag{
+			Name:  "profile",
+			Usage: "enable pprof remote profiler on localhost:6060",
+		},
+		cli.IntFlag{
+			Name:  "profile-port",
+			Value: 6060,
+			Usage: "port for the pprof profiler",
+		},
+		cli.BoolFlag{
+			Name:  "enable-metrics",
+			Usage: "enable metrics endpoint for the servier on localhost:9090",
+		},
+		cli.IntFlag{
+			Name:  "metrics-port",
+			Value: 9090,
+			Usage: "port for the metrics endpoint",
+		},
+	}
+
+	sort.Sort(cli.FlagsByName(app.Flags))
+	sort.Sort(cli.FlagsByName(configCommand.Flags))
+
+	app.Commands = []cli.Command{
+		configCommand,
+	}
+
+	app.Before = func(c *cli.Context) error {
+		// Load the configuration file.
+		config := c.App.Metadata["config"].(*server.Config)
+		if err := mergeConfig(config, c); err != nil {
+			return err
+		}
+
+		if err := validateConfig(config); err != nil {
+			return err
+		}
+
+		cf := &logrus.TextFormatter{
+			TimestampFormat: "2006-01-02 15:04:05.000000000Z07:00",
+			FullTimestamp:   true,
+		}
+
+		logrus.SetFormatter(cf)
+
+		if loglevel := c.GlobalString("log-level"); loglevel != "" {
+			level, err := logrus.ParseLevel(loglevel)
+			if err != nil {
+				return err
+			}
+
+			logrus.SetLevel(level)
+		}
+
+		if path := c.GlobalString("log"); path != "" {
+			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0666)
+			if err != nil {
+				return err
+			}
+			logrus.SetOutput(f)
+		}
+
+		switch c.GlobalString("log-format") {
+		case "text":
+			// retain logrus's default.
+		case "json":
+			logrus.SetFormatter(new(logrus.JSONFormatter))
+		default:
+			return fmt.Errorf("unknown log-format %q", c.GlobalString("log-format"))
+		}
+
+		return nil
+	}
+
+	app.Action = func(c *cli.Context) error {
+		if c.GlobalBool("profile") {
+			profilePort := c.GlobalInt("profile-port")
+			profileEndpoint := fmt.Sprintf("localhost:%v", profilePort)
+			go func() {
+				http.ListenAndServe(profileEndpoint, nil)
+			}()
+		}
+
+		args := c.Args()
+		if len(args) > 0 {
+			for _, command := range app.Commands {
+				if args[0] == command.Name {
+					break
+				}
+			}
+			return fmt.Errorf("command %q not supported", args[0])
+		}
+
+		config := c.App.Metadata["config"].(*server.Config)
+
+		if !config.SELinux {
+			selinux.SetDisabled()
+		}
+
+		if _, err := os.Stat(config.Runtime); os.IsNotExist(err) {
+			// path to runtime does not exist
+			return fmt.Errorf("invalid --runtime value %q", err)
+		}
+
+		// Remove the socket if it already exists
+		if _, err := os.Stat(config.Listen); err == nil {
+			if err := os.Remove(config.Listen); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+		lis, err := net.Listen("unix", config.Listen)
+		if err != nil {
+			logrus.Fatalf("failed to listen: %v", err)
+		}
+
+		s := grpc.NewServer()
+
+		service, err := server.New(config)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if c.GlobalBool("enable-metrics") {
+			metricsPort := c.GlobalInt("metrics-port")
+			me, err := service.CreateMetricsEndpoint()
+			if err != nil {
+				logrus.Fatalf("Failed to create metrics endpoint: %v", err)
+			}
+			l, err := net.Listen("tcp", fmt.Sprintf(":%v", metricsPort))
+			if err != nil {
+				logrus.Fatalf("Failed to create listener for metrics: %v", err)
+			}
+			go func() {
+				if err := http.Serve(l, me); err != nil {
+					logrus.Fatalf("Failed to serve metrics endpoint: %v", err)
+				}
+			}()
+		}
+
+		runtime.RegisterRuntimeServiceServer(s, service)
+		runtime.RegisterImageServiceServer(s, service)
+
+		// after the daemon is done setting up we can notify systemd api
+		notifySystem()
+
+		go func() {
+			service.StartExitMonitor()
+		}()
+
+		m := cmux.New(lis)
+		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
+		httpL := m.Match(cmux.HTTP1Fast())
+
+		infoMux := service.GetInfoMux()
+		srv := &http.Server{
+			Handler:     infoMux,
+			ReadTimeout: 5 * time.Second,
+		}
+
+		graceful := false
+		catchShutdown(s, service, srv, &graceful)
+
+		go s.Serve(grpcL)
+		go srv.Serve(httpL)
+
+		serverCloseCh := make(chan struct{})
+		go func() {
+			defer close(serverCloseCh)
+			if err := m.Serve(); err != nil {
+				if graceful && strings.Contains(strings.ToLower(err.Error()), "use of closed network connection") {
+					err = nil
+				} else {
+					logrus.Errorf("Failed to serve grpc grpc request: %v", err)
+				}
+			}
+		}()
+
+		// TODO(runcom): enable this after https://github.com/kubernetes/kubernetes/pull/51377
+		//streamServerCloseCh := service.StreamingServerCloseChan()
+		serverExitMonitorCh := service.ExitMonitorCloseChan()
+		select {
+		// TODO(runcom): enable this after https://github.com/kubernetes/kubernetes/pull/51377
+		//case <-streamServerCloseCh:
+		case <-serverExitMonitorCh:
+		case <-serverCloseCh:
+		}
+
+		service.Shutdown()
+
+		// TODO(runcom): enable this after https://github.com/kubernetes/kubernetes/pull/51377
+		//<-streamServerCloseCh
+		//logrus.Debug("closed stream server")
+		<-serverExitMonitorCh
+		logrus.Debug("closed exit monitor")
+		<-serverCloseCh
+		logrus.Debug("closed main server")
+
+		return nil
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}