From b0286d6b43ebec367c0d9ed87bc6566d76ece8f8 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Tue, 9 Jun 2020 17:10:37 -0400 Subject: Implement pod-network-reload This adds a new command, 'podman network reload', to reload the networks of existing containers, forcing recreation of firewall rules after e.g. `firewall-cmd --reload` wipes them out. Under the hood, this works by calling CNI to tear down the existing network, then recreate it using identical settings. We request that CNI preserve the old IP and MAC address in most cases (where the container only had 1 IP/MAC), but there will be some downtime inherent to the teardown/bring-up approach. The architecture of CNI doesn't really make doing this without downtime easy (or maybe even possible...). At present, this only works for root Podman, and only locally. I don't think there is much of a point to adding remote support (this is very much a local debugging command), but I think adding rootless support (to kill/recreate slirp4netns) could be valuable. Signed-off-by: Matthew Heon Signed-off-by: Paul Holzinger --- cmd/podman/networks/reload.go | 69 ++++++++++++++++++++ docs/source/markdown/podman-network-reload.1.md | 62 ++++++++++++++++++ docs/source/markdown/podman-network.1.md | 17 ++--- docs/source/network.rst | 2 + libpod/container_api.go | 26 ++++++++ libpod/container_internal_linux.go | 13 ++++ libpod/container_internal_unsupported.go | 4 ++ libpod/networking_linux.go | 85 +++++++++++++++++++++++-- libpod/networking_unsupported.go | 9 ++- pkg/domain/entities/engine_container.go | 1 + pkg/domain/entities/network.go | 13 ++++ pkg/domain/infra/abi/network.go | 20 ++++++ pkg/domain/infra/tunnel/network.go | 4 ++ test/system/500-networking.bats | 64 +++++++++++++++++++ 14 files changed, 376 insertions(+), 13 deletions(-) create mode 100644 cmd/podman/networks/reload.go create mode 100644 docs/source/markdown/podman-network-reload.1.md diff --git a/cmd/podman/networks/reload.go b/cmd/podman/networks/reload.go new file mode 100644 index 000000000..16655c18c --- /dev/null +++ b/cmd/podman/networks/reload.go @@ -0,0 +1,69 @@ +package network + +import ( + "fmt" + + "github.com/containers/podman/v2/cmd/podman/common" + "github.com/containers/podman/v2/cmd/podman/registry" + "github.com/containers/podman/v2/cmd/podman/utils" + "github.com/containers/podman/v2/cmd/podman/validate" + "github.com/containers/podman/v2/pkg/domain/entities" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var ( + networkReloadDescription = `reload container networks, recreating firewall rules` + networkReloadCommand = &cobra.Command{ + Use: "reload [options] [CONTAINER...]", + Short: "Reload firewall rules for one or more containers", + Long: networkReloadDescription, + RunE: networkReload, + Args: func(cmd *cobra.Command, args []string) error { + return validate.CheckAllLatestAndCIDFile(cmd, args, false, false) + }, + ValidArgsFunction: common.AutocompleteContainers, + Example: `podman network reload --latest + podman network reload 3c13ef6dd843 + podman network reload test1 test2`, + Annotations: map[string]string{ + registry.ParentNSRequired: "", + }, + } +) + +var ( + reloadOptions entities.NetworkReloadOptions +) + +func reloadFlags(flags *pflag.FlagSet) { + flags.BoolVarP(&reloadOptions.All, "all", "a", false, "Reload network configuration of all containers") +} + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Mode: []entities.EngineMode{entities.ABIMode}, + Command: networkReloadCommand, + Parent: networkCmd, + }) + reloadFlags(networkReloadCommand.Flags()) + validate.AddLatestFlag(networkReloadCommand, &reloadOptions.Latest) +} + +func networkReload(cmd *cobra.Command, args []string) error { + responses, err := registry.ContainerEngine().NetworkReload(registry.Context(), args, reloadOptions) + if err != nil { + return err + } + + var errs utils.OutputErrors + for _, r := range responses { + if r.Err == nil { + fmt.Println(r.Id) + } else { + errs = append(errs, r.Err) + } + } + + return errs.PrintErrors() +} diff --git a/docs/source/markdown/podman-network-reload.1.md b/docs/source/markdown/podman-network-reload.1.md new file mode 100644 index 000000000..dd8047297 --- /dev/null +++ b/docs/source/markdown/podman-network-reload.1.md @@ -0,0 +1,62 @@ +% podman-network-reload(1) + +## NAME +podman\-network\-reload - Reload network configuration for containers + +## SYNOPSIS +**podman network reload** [*options*] [*container...*] + +## DESCRIPTION +Reload one or more container network configurations. + +Rootful Podman relies on iptables rules in order to provide network connectivity. If the iptables rules are deleted, +this happens for example with `firewall-cmd --reload`, the container loses network connectivity. This command restores +the network connectivity. + +This command is not available for rootless users since rootless containers are not affected by such connectivity problems. + +## OPTIONS +#### **--all**, **-a** + +Reload network configuration of all containers. + +#### **--latest**, **-l** + +Instead of providing the container name or ID, use the last created container. If you use methods other than Podman +to run containers such as CRI-O, the last started container could be from either of those methods. + +The latest option is not supported on the remote client. + +## EXAMPLE + +Reload the network configuration after a firewall reload. + +``` +# podman run -p 80:80 -d nginx +b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c +# curl 127.0.0.1 +works +# sudo firewall-cmd --reload +success +# curl 127.0.0.1 +hangs +# podman network reload b1b538e8bc40 +b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c +# curl 127.0.0.1 +works +``` + +Reload the network configuration for all containers. + +``` +# podman network reload --all +b1b538e8bc4078fc3ee1c95b666ebc7449b9a97bacd15bcbe464a29e1be59c1c +fe7e8eca56f844ec33af10f0aa3b31b44a172776e3277b9550a623ed5d96e72b +``` + + +## SEE ALSO +podman(1), podman-network(1) + +## HISTORY +December 2020, Originally compiled by Paul Holzinger diff --git a/docs/source/markdown/podman-network.1.md b/docs/source/markdown/podman-network.1.md index d21b200d9..d11f0d7b6 100644 --- a/docs/source/markdown/podman-network.1.md +++ b/docs/source/markdown/podman-network.1.md @@ -11,14 +11,15 @@ The network command manages CNI networks for Podman. It is not supported for roo ## COMMANDS -| Command | Man Page | Description | -| ------- | --------------------------------------------------- | ---------------------------------------------------------------------------- | -| connect | [podman-network-connect(1)](podman-network-connect.1.md)| Connect a container to a network| -| create | [podman-network-create(1)](podman-network-create.1.md)| Create a Podman CNI network| -| disconnect | [podman-network-disconnect(1)](podman-network-disconnect.1.md)| Disconnect a container from a network| -| inspect | [podman-network-inspect(1)](podman-network-inspect.1.md)| Displays the raw CNI network configuration for one or more networks| -| ls | [podman-network-ls(1)](podman-network-ls.1.md)| Display a summary of CNI networks | -| rm | [podman-network-rm(1)](podman-network-rm.1.md)| Remove one or more CNI networks | +| Command | Man Page | Description | +| ---------- | -------------------------------------------------------------- | ------------------------------------------------------------------- | +| connect | [podman-network-connect(1)](podman-network-connect.1.md) | Connect a container to a network | +| create | [podman-network-create(1)](podman-network-create.1.md) | Create a Podman CNI network | +| disconnect | [podman-network-disconnect(1)](podman-network-disconnect.1.md) | Disconnect a container from a network | +| inspect | [podman-network-inspect(1)](podman-network-inspect.1.md) | Displays the raw CNI network configuration for one or more networks | +| ls | [podman-network-ls(1)](podman-network-ls.1.md) | Display a summary of CNI networks | +| reload | [podman-network-reload(1)](podman-network-reload.1.md) | Reload network configuration for containers | +| rm | [podman-network-rm(1)](podman-network-rm.1.md) | Remove one or more CNI networks | ## SEE ALSO podman(1) diff --git a/docs/source/network.rst b/docs/source/network.rst index 0414c0880..2ecb97858 100644 --- a/docs/source/network.rst +++ b/docs/source/network.rst @@ -11,4 +11,6 @@ Network :doc:`ls ` network list +:doc:`reload ` network reload + :doc:`rm ` network rm diff --git a/libpod/container_api.go b/libpod/container_api.go index 6a7ddc421..1b33f16b4 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -639,6 +639,32 @@ func (c *Container) Sync() error { return nil } +// ReloadNetwork reconfigures the container's network. +// Technically speaking, it will tear down and then reconfigure the container's +// network namespace, which will result in all firewall rules being recreated. +// It is mostly intended to be used in cases where the system firewall has been +// reloaded, and existing rules have been wiped out. It is expected that some +// downtime will result, as the rules are destroyed as part of this process. +// At present, this only works on root containers; it may be expanded to restart +// slirp4netns in the future to work with rootless containers as well. +// Requires that the container must be running or created. +func (c *Container) ReloadNetwork() error { + if !c.batched { + c.lock.Lock() + defer c.lock.Unlock() + + if err := c.syncContainer(); err != nil { + return err + } + } + + if !c.ensureState(define.ContainerStateCreated, define.ContainerStateRunning) { + return errors.Wrapf(define.ErrCtrStateInvalid, "cannot reload network unless container network has been configured") + } + + return c.reloadNetwork() +} + // Refresh is DEPRECATED and REMOVED. func (c *Container) Refresh(ctx context.Context) error { // This has been deprecated for a long while, and is in the process of diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 56575c195..a9a789ad8 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -230,6 +230,19 @@ func (c *Container) cleanupNetwork() error { return nil } +// reloadNetwork reloads the network for the given container, recreating +// firewall rules. +func (c *Container) reloadNetwork() error { + result, err := c.runtime.reloadContainerNetwork(c) + if err != nil { + return err + } + + c.state.NetworkStatus = result + + return c.save() +} + func (c *Container) getUserOverrides() *lookup.Overrides { var hasPasswdFile, hasGroupFile bool overrides := lookup.Overrides{} diff --git a/libpod/container_internal_unsupported.go b/libpod/container_internal_unsupported.go index c22e9a4a4..7f6fc9ec9 100644 --- a/libpod/container_internal_unsupported.go +++ b/libpod/container_internal_unsupported.go @@ -50,6 +50,10 @@ func (c *Container) cleanupOverlayMounts() error { return nil } +func (c *Container) reloadNetwork() error { + return define.ErrNotImplemented +} + func (c *Container) getUserOverrides() *lookup.Overrides { return nil } diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 15e470c80..c7db95eb3 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "sort" "strings" "syscall" @@ -740,8 +741,9 @@ func (r *Runtime) closeNetNS(ctr *Container) error { return nil } -// Tear down a network namespace, undoing all state associated with it. -func (r *Runtime) teardownNetNS(ctr *Container) error { +// Tear down a container's CNI network configuration, but do not tear down the +// namespace itself. +func (r *Runtime) teardownCNI(ctr *Container) error { if ctr.state.NetNS == nil { // The container has no network namespace, we're set return nil @@ -780,6 +782,19 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", ctr.ID()) } } + return nil +} + +// Tear down a network namespace, undoing all state associated with it. +func (r *Runtime) teardownNetNS(ctr *Container) error { + if err := r.teardownCNI(ctr); err != nil { + return err + } + + networks, _, err := ctr.networks() + if err != nil { + return err + } // CNI-in-slirp4netns if rootless.IsRootless() && len(networks) != 0 { @@ -820,6 +835,68 @@ func getContainerNetNS(ctr *Container) (string, error) { return "", nil } +// Reload only works with containers with a configured network. +// It will tear down, and then reconfigure, the network of the container. +// This is mainly used when a reload of firewall rules wipes out existing +// firewall configuration. +// Efforts will be made to preserve MAC and IP addresses, but this only works if +// the container only joined a single CNI network, and was only assigned a +// single MAC or IP. +// Only works on root containers at present, though in the future we could +// extend this to stop + restart slirp4netns +func (r *Runtime) reloadContainerNetwork(ctr *Container) ([]*cnitypes.Result, error) { + if ctr.state.NetNS == nil { + return nil, errors.Wrapf(define.ErrCtrStateInvalid, "container %s network is not configured, refusing to reload", ctr.ID()) + } + if rootless.IsRootless() || ctr.config.NetMode.IsSlirp4netns() { + return nil, errors.Wrapf(define.ErrRootless, "network reload only supported for root containers") + } + + logrus.Infof("Going to reload container %s network", ctr.ID()) + + var requestedIP net.IP + var requestedMAC net.HardwareAddr + // Set requested IP and MAC address, if possible. + if len(ctr.state.NetworkStatus) == 1 { + result := ctr.state.NetworkStatus[0] + if len(result.IPs) == 1 { + resIP := result.IPs[0] + + requestedIP = resIP.Address.IP + ctr.requestedIP = requestedIP + logrus.Debugf("Going to preserve container %s IP address %s", ctr.ID(), ctr.requestedIP.String()) + + if resIP.Interface != nil && *resIP.Interface < len(result.Interfaces) && *resIP.Interface >= 0 { + var err error + requestedMAC, err = net.ParseMAC(result.Interfaces[*resIP.Interface].Mac) + if err != nil { + return nil, errors.Wrapf(err, "error parsing container %s MAC address %s", ctr.ID(), result.Interfaces[*resIP.Interface].Mac) + } + ctr.requestedMAC = requestedMAC + logrus.Debugf("Going to preserve container %s MAC address %s", ctr.ID(), ctr.requestedMAC.String()) + } + } + } + + err := r.teardownCNI(ctr) + if err != nil { + // teardownCNI will error if the iptables rules do not exists and this is the case after + // a firewall reload. The purpose of network reload is to recreate the rules if they do + // not exists so we should not log this specific error as error. This would confuse users otherwise. + b, rerr := regexp.MatchString("Couldn't load target `CNI-[a-f0-9]{24}':No such file or directory", err.Error()) + if rerr == nil && !b { + logrus.Error(err) + } else { + logrus.Info(err) + } + } + + // teardownCNI will clean the requested IP and MAC so we need to set them again + ctr.requestedIP = requestedIP + ctr.requestedMAC = requestedMAC + return r.configureNetNS(ctr, ctr.state.NetNS) +} + func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { var netStats *netlink.LinkStatistics // rootless v2 cannot seem to resolve its network connection to @@ -983,12 +1060,12 @@ func resultToBasicNetworkConfig(result *cnitypes.Result) (define.InspectBasicNet config.IPAddress = ctrIP.Address.IP.String() config.IPPrefixLen = size config.Gateway = ctrIP.Gateway.String() - if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface > 0 { + if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface >= 0 { config.MacAddress = result.Interfaces[*ctrIP.Interface].Mac } case ctrIP.Version == "4" && config.IPAddress != "": config.SecondaryIPAddresses = append(config.SecondaryIPAddresses, ctrIP.Address.String()) - if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface > 0 { + if ctrIP.Interface != nil && *ctrIP.Interface < len(result.Interfaces) && *ctrIP.Interface >= 0 { config.AdditionalMacAddresses = append(config.AdditionalMacAddresses, result.Interfaces[*ctrIP.Interface].Mac) } case ctrIP.Version == "6" && config.IPAddress == "": diff --git a/libpod/networking_unsupported.go b/libpod/networking_unsupported.go index 76bb01424..9e5c4adde 100644 --- a/libpod/networking_unsupported.go +++ b/libpod/networking_unsupported.go @@ -2,7 +2,10 @@ package libpod -import "github.com/containers/podman/v2/libpod/define" +import ( + cnitypes "github.com/containernetworking/cni/pkg/types/current" + "github.com/containers/podman/v2/libpod/define" +) func (r *Runtime) setupRootlessNetNS(ctr *Container) error { return define.ErrNotImplemented @@ -28,6 +31,10 @@ func (c *Container) getContainerNetworkInfo() (*define.InspectNetworkSettings, e return nil, define.ErrNotImplemented } +func (r *Runtime) reloadContainerNetwork(ctr *Container) ([]*cnitypes.Result, error) { + return nil, define.ErrNotImplemented +} + func getCNINetworksDir() (string, error) { return "", define.ErrNotImplemented } diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index df7da616a..a4f6c08e9 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -55,6 +55,7 @@ type ContainerEngine interface { NetworkDisconnect(ctx context.Context, networkname string, options NetworkDisconnectOptions) error NetworkInspect(ctx context.Context, namesOrIds []string, options InspectOptions) ([]NetworkInspectReport, []error, error) NetworkList(ctx context.Context, options NetworkListOptions) ([]*NetworkListReport, error) + NetworkReload(ctx context.Context, names []string, options NetworkReloadOptions) ([]*NetworkReloadReport, error) NetworkRm(ctx context.Context, namesOrIds []string, options NetworkRmOptions) ([]*NetworkRmReport, error) PlayKube(ctx context.Context, path string, opts PlayKubeOptions) (*PlayKubeReport, error) PodCreate(ctx context.Context, opts PodCreateOptions) (*PodCreateReport, error) diff --git a/pkg/domain/entities/network.go b/pkg/domain/entities/network.go index 65a110fd9..b76bfcac7 100644 --- a/pkg/domain/entities/network.go +++ b/pkg/domain/entities/network.go @@ -22,6 +22,19 @@ type NetworkListReport struct { // NetworkInspectReport describes the results from inspect networks type NetworkInspectReport map[string]interface{} +// NetworkReloadOptions describes options for reloading container network +// configuration. +type NetworkReloadOptions struct { + All bool + Latest bool +} + +// NetworkReloadReport describes the results of reloading a container network. +type NetworkReloadReport struct { + Id string + Err error +} + // NetworkRmOptions describes options for removing networks type NetworkRmOptions struct { Force bool diff --git a/pkg/domain/infra/abi/network.go b/pkg/domain/infra/abi/network.go index 6a219edd5..e5ecf5c72 100644 --- a/pkg/domain/infra/abi/network.go +++ b/pkg/domain/infra/abi/network.go @@ -60,6 +60,26 @@ func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []stri return rawCNINetworks, errs, nil } +func (ic *ContainerEngine) NetworkReload(ctx context.Context, names []string, options entities.NetworkReloadOptions) ([]*entities.NetworkReloadReport, error) { + ctrs, err := getContainersByContext(options.All, options.Latest, names, ic.Libpod) + if err != nil { + return nil, err + } + + reports := make([]*entities.NetworkReloadReport, 0, len(ctrs)) + for _, ctr := range ctrs { + report := new(entities.NetworkReloadReport) + report.Id = ctr.ID() + report.Err = ctr.ReloadNetwork() + if options.All && errors.Cause(report.Err) == define.ErrCtrStateInvalid { + continue + } + reports = append(reports, report) + } + + return reports, nil +} + func (ic *ContainerEngine) NetworkRm(ctx context.Context, namesOrIds []string, options entities.NetworkRmOptions) ([]*entities.NetworkRmReport, error) { reports := []*entities.NetworkRmReport{} diff --git a/pkg/domain/infra/tunnel/network.go b/pkg/domain/infra/tunnel/network.go index 10ae03045..4845980f6 100644 --- a/pkg/domain/infra/tunnel/network.go +++ b/pkg/domain/infra/tunnel/network.go @@ -35,6 +35,10 @@ func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []stri return reports, errs, nil } +func (ic *ContainerEngine) NetworkReload(ctx context.Context, names []string, options entities.NetworkReloadOptions) ([]*entities.NetworkReloadReport, error) { + return nil, errors.New("not implemented") +} + func (ic *ContainerEngine) NetworkRm(ctx context.Context, namesOrIds []string, options entities.NetworkRmOptions) ([]*entities.NetworkRmReport, error) { reports := make([]*entities.NetworkRmReport, 0, len(namesOrIds)) for _, name := range namesOrIds { diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index 44cc731cf..a824ebcd7 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -116,4 +116,68 @@ load helpers fi } +@test "podman network reload" { + skip_if_remote "podman network reload does not have remote support" + skip_if_rootless "podman network reload does not work rootless" + + random_1=$(random_string 30) + HOST_PORT=12345 + SERVER=http://127.0.0.1:$HOST_PORT + + # Create a test file with random content + INDEX1=$PODMAN_TMPDIR/hello.txt + echo $random_1 > $INDEX1 + + # Bind-mount this file with a different name to a container running httpd + run_podman run -d --name myweb -p "$HOST_PORT:80" \ + -v $INDEX1:/var/www/index.txt \ + -w /var/www \ + $IMAGE /bin/busybox-extras httpd -f -p 80 + cid=$output + + run_podman inspect $cid --format "{{.NetworkSettings.IPAddress}}" + ip="$output" + run_podman inspect $cid --format "{{.NetworkSettings.MacAddress}}" + mac="$output" + + # Verify http contents: curl from localhost + run curl -s $SERVER/index.txt + is "$output" "$random_1" "curl 127.0.0.1:/index.txt" + + # flush the CNI iptables here + run iptables -t nat -F CNI-HOSTPORT-DNAT + + # check that we cannot curl (timeout after 5 sec) + run timeout 5 curl -s $SERVER/index.txt + if [ "$status" -ne 124 ]; then + die "curl did not timeout, status code: $status" + fi + + # reload the network to recreate the iptables rules + run_podman network reload $cid + is "$output" "$cid" "Output does not match container ID" + + # check that we still have the same mac and ip + run_podman inspect $cid --format "{{.NetworkSettings.IPAddress}}" + is "$output" "$ip" "IP address changed after podman network reload" + run_podman inspect $cid --format "{{.NetworkSettings.MacAddress}}" + is "$output" "$mac" "MAC address changed after podman network reload" + + # check that we can still curl + run curl -s $SERVER/index.txt + is "$output" "$random_1" "curl 127.0.0.1:/index.txt" + + # make sure --all is working and that this + # cmd also works if the iptables still exists + run_podman network reload --all + is "$output" "$cid" "Output does not match container ID" + + # check that we can still curl + run curl -s $SERVER/index.txt + is "$output" "$random_1" "curl 127.0.0.1:/index.txt" + + # cleanup the container + run_podman rm -f $cid +} + # vim: filetype=sh -- cgit v1.2.3-54-g00ecf