diff --git a/doc/how-to-use.md b/doc/how-to-use.md index ee6b74ed6..984b3b956 100644 --- a/doc/how-to-use.md +++ b/doc/how-to-use.md @@ -619,3 +619,7 @@ When using `--multus-conf-file=auto` you may also care to specify a `binDir` in Sometimes, you may wish to not have the entrypoint copy the binary file onto the host. Potentially, you have another way to copy in a specific version of Multus, for example. By default, it's always copied, but you may disable the copy with: --skip-multus-binary-copy=true + +If you wish to have auto configuration use the `readinessindicatorfile` in the configuration, you can use the `--readiness-indicator-file` to express which file should be used as the readiness indicator. + + --readiness-indicator-file=/path/to/file diff --git a/images/entrypoint.sh b/images/entrypoint.sh index 5c9a7336f..1d3fd787f 100755 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -28,6 +28,7 @@ MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig" MULTUS_NAMESPACE_ISOLATION=false MULTUS_LOG_LEVEL="" MULTUS_LOG_FILE="" +MULTUS_READINESS_INDICATOR_FILE="" OVERRIDE_NETWORK_NAME=false MULTUS_CLEANUP_CONFIG_ON_EXIT=false RESTART_CRIO=false @@ -61,6 +62,7 @@ function usage() echo -e "\t--override-network-name=false (used only with --multus-conf-file=auto)" echo -e "\t--cleanup-config-on-exit=false (used only with --multus-conf-file=auto)" echo -e "\t--rename-conf-file=false (used only with --multus-conf-file=auto)" + echo -e "\t--readiness-indicator-file=$MULTUS_READINESS_INDICATOR_FILE (used only with --multus-conf-file=auto)" echo -e "\t--additional-bin-dir=$ADDITIONAL_BIN_DIR (adds binDir option to configuration, used only with --multus-conf-file=auto)" echo -e "\t--restart-crio=false (restarts CRIO after config file is generated)" } @@ -137,6 +139,9 @@ while [ "$1" != "" ]; do --skip-multus-binary-copy) SKIP_BINARY_COPY=$VALUE ;; + --readiness-indicator-file) + MULTUS_READINESS_INDICATOR_FILE=$VALUE + ;; *) warn "unknown parameter \"$PARAM\"" ;; @@ -305,6 +310,12 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then ADDITIONAL_BIN_DIR_STRING="\"binDir\": \"$ADDITIONAL_BIN_DIR\"," fi + + READINESS_INDICATOR_FILE_STRING="" + if [ ! -z "${MULTUS_READINESS_INDICATOR_FILE// }" ]; then + READINESS_INDICATOR_FILE_STRING="\"readinessindicatorfile\": \"$MULTUS_READINESS_INDICATOR_FILE\"," + fi + if [ "$OVERRIDE_NETWORK_NAME" == "true" ]; then MASTER_PLUGIN_NET_NAME="$(cat $MULTUS_AUTOCONF_DIR/$MASTER_PLUGIN | \ python -c 'import json,sys;print json.load(sys.stdin)["name"]')" @@ -324,6 +335,7 @@ if [ "$MULTUS_CONF_FILE" == "auto" ]; then $LOG_LEVEL_STRING $LOG_FILE_STRING $ADDITIONAL_BIN_DIR_STRING + $READINESS_INDICATOR_FILE_STRING "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST", "delegates": [ $MASTER_PLUGIN_JSON @@ -363,8 +375,21 @@ if [ "$MULTUS_CLEANUP_CONFIG_ON_EXIT" == true ]; then while true; do # Check and see if the original master plugin configuration exists... if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then - log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Performing cleanup..." - cleanup + log "Master plugin @ $MASTER_PLUGIN_LOCATION has been deleted. Allowing 45 seconds for its restoration..." + sleep 10 + for i in {1..35} + do + if [ -f "$MASTER_PLUGIN_LOCATION" ]; then + log "Master plugin @ $MASTER_PLUGIN_LOCATION was restored. Regenerating given configuration." + break + fi + sleep 1 + done + + if [ ! -f "$MASTER_PLUGIN_LOCATION" ]; then + log "Master plugin @ $MASTER_PLUGIN_LOCATION has not been restored, beginning regeneration." + cleanup + fi generateMultusConf log "Continuing watch loop after configuration regeneration..." fi diff --git a/multus/multus.go b/multus/multus.go index 64b403110..02c0b2e53 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -48,12 +48,8 @@ var version = "master@git" var commit = "unknown commit" var date = "unknown date" -var defaultReadinessBackoff = wait.Backoff{ - Steps: 4, - Duration: 250 * time.Millisecond, - Factor: 4.0, - Jitter: 0.1, -} +var pollDuration = 1000 * time.Millisecond +var pollTimeout = 45 * time.Second func printVersionString() string { return fmt.Sprintf("multus-cni version:%s, commit:%s, date:%s", @@ -359,32 +355,40 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN return nil } +func cmdErr(k8sArgs *types.K8sArgs, format string, args ...interface{}) error { + prefix := "Multus: " + if k8sArgs != nil { + prefix += fmt.Sprintf("[%s/%s]: ", k8sArgs.K8S_POD_NAMESPACE, k8sArgs.K8S_POD_NAME) + } + return logging.Errorf(prefix+format, args...) +} + func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) { n, err := types.LoadNetConf(args.StdinData) logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient) if err != nil { - return nil, logging.Errorf("Multus: error loading netconf: %v", err) + return nil, cmdErr(nil, "error loading netconf: %v", err) } k8sArgs, err := k8s.GetK8sArgs(args) if err != nil { - return nil, logging.Errorf("Multus: error getting k8s args: %v", err) + return nil, cmdErr(nil, "error getting k8s args: %v", err) } - wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) { - _, err := os.Stat(n.ReadinessIndicatorFile) - switch { - case err == nil: - return true, nil - default: - return false, nil + if n.ReadinessIndicatorFile != "" { + err := wait.PollImmediate(pollDuration, pollTimeout, func() (bool, error) { + _, err := os.Stat(n.ReadinessIndicatorFile) + return err == nil, nil + }) + if err != nil { + return nil, cmdErr(k8sArgs, "PollImmediate error waiting for ReadinessIndicatorFile: %v", err) } - }) + } if n.ClusterNetwork != "" { err = k8s.GetDefaultNetworks(k8sArgs, n, kubeClient) if err != nil { - return nil, logging.Errorf("Multus: failed to get clusterNetwork/defaultNetworks: %v", err) + return nil, cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err) } // First delegate is always the master plugin n.Delegates[0].MasterPlugin = true @@ -392,12 +396,12 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn _, kc, err := k8s.TryLoadPodDelegates(k8sArgs, n, kubeClient) if err != nil { - return nil, logging.Errorf("Multus: error loading k8s delegates k8s args: %v", err) + return nil, cmdErr(k8sArgs, "error loading k8s delegates k8s args: %v", err) } // cache the multus config if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil { - return nil, logging.Errorf("Multus: error saving the delegates: %v", err) + return nil, cmdErr(k8sArgs, "error saving the delegates: %v", err) } var result, tmpResult cnitypes.Result @@ -417,7 +421,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn } // Ignore errors; DEL must be idempotent anyway _ = delPlugins(exec, args.IfName, n.Delegates, idx, rt, n.BinDir) - return nil, logging.Errorf("Multus: error adding pod to network %q: %v", netName, err) + return nil, cmdErr(k8sArgs, "error adding container to network %q: %v", netName, err) } // Remove gateway from routing table if the gateway is not used @@ -438,7 +442,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if deletegateway { tmpResult, err = netutils.DeleteDefaultGW(args, ifName, &tmpResult) if err != nil { - return nil, logging.Errorf("Multus: Err in deleting gateway: %v", err) + return nil, cmdErr(k8sArgs, "error deleting default gateway: %v", err) } } @@ -446,7 +450,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if adddefaultgateway { tmpResult, err = netutils.SetDefaultGW(args, ifName, delegate.GatewayRequest, &tmpResult) if err != nil { - return nil, logging.Errorf("Multus: Err in setting default gateway: %v", err) + return nil, cmdErr(k8sArgs, "error setting default gateway: %v", err) } } @@ -460,7 +464,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) { delegateNetStatus, err := types.LoadNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin) if err != nil { - return nil, logging.Errorf("Multus: error setting network status: %v", err) + return nil, cmdErr(k8sArgs, "error setting network status: %v", err) } netStatus = append(netStatus, delegateNetStatus) @@ -473,7 +477,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if !types.CheckSystemNamespaces(kc.Podnamespace, n.SystemNamespaces) { err = k8s.SetNetworkStatus(kubeClient, k8sArgs, netStatus, n) if err != nil { - return nil, logging.Errorf("Multus: error setting the networks status: %v", err) + return nil, cmdErr(k8sArgs, "error setting the networks status: %v", err) } } } @@ -511,7 +515,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err netnsfound = false logging.Debugf("cmdDel: WARNING netns may not exist, netns: %s, err: %s", args.Netns, err) } else { - return logging.Errorf("Multus: failed to open netns %q: %v", netns, err) + return cmdErr(nil, "failed to open netns %q: %v", netns, err) } } @@ -521,7 +525,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err k8sArgs, err := k8s.GetK8sArgs(args) if err != nil { - return logging.Errorf("Multus: error getting k8s args: %v", err) + return cmdErr(nil, "error getting k8s args: %v", err) } // Read the cache to get delegates json for the pod @@ -532,7 +536,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err if in.ClusterNetwork != "" { err = k8s.GetDefaultNetworks(k8sArgs, in, kubeClient) if err != nil { - return logging.Errorf("Multus: failed to get clusterNetwork/defaultNetworks: %v", err) + return cmdErr(k8sArgs, "failed to get clusterNetwork/defaultNetworks: %v", err) } // First delegate is always the master plugin in.Delegates[0].MasterPlugin = true @@ -543,18 +547,18 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err if err != nil { if len(in.Delegates) == 0 { // No delegate available so send error - return logging.Errorf("Multus: failed to get delegates: %v", err) + return cmdErr(k8sArgs, "failed to get delegates: %v", err) } // Get clusterNetwork before, so continue to delete logging.Errorf("Multus: failed to get delegates: %v, but continue to delete clusterNetwork", err) } } else { - return logging.Errorf("Multus: error reading the delegates: %v", err) + return cmdErr(k8sArgs, "error reading the delegates: %v", err) } } else { defer os.Remove(path) if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil { - return logging.Errorf("Multus: failed to load netconf: %v", err) + return cmdErr(k8sArgs, "failed to load netconf: %v", err) } // check plugins field and enable ConfListPlugin if there is for _, v := range in.Delegates {