Merge pull request #10118 from azhao155/yzhao/fix/flacknessDocker-ENV

avoid race condition in bootstrap certs for parallel runs
kubernetes · Jan 13, 2021 · 31aa5c1 · 31aa5c1
2 parents d3ad8ca + e933166
commit 31aa5c1
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 1 deletion.
diff --git a/go.mod b/go.mod
@@ -46,6 +46,7 @@ require (
 	github.com/johanneswuerbach/nfsexports v0.0.0-20200318065542-c48c3734757f
 	github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c
 	github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect
+	github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/mutex v0.0.0-20180619145857-d21b13acf4bf
 	github.com/juju/retry v0.0.0-20180821225755-9058e192b216 // indirect

diff --git a/go.sum b/go.sum
@@ -627,6 +627,8 @@ github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN
 github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA=
 github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc=
 github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
+github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b h1:FQ7+9fxhyp82ks9vAuyPzG0/vVbWwMwLJ+P6yJI5FN8=
+github.com/juju/fslock v0.0.0-20160525022230-4d5c94c67b4b/go.mod h1:HMcgvsgd0Fjj4XXDkbjdmlbI505rUPBs6WBMYg2pXks=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI=
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/mutex v0.0.0-20180619145857-d21b13acf4bf h1:2d3cilQly1OpAfZcn4QRuwDOdVoHsM4cDTkcKbmO760=

diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go
@@ -40,6 +40,7 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/docker/machine/libmachine/swarm"
 	"github.com/docker/machine/libmachine/version"
+	"github.com/juju/fslock"
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -71,6 +72,7 @@ func NewAPIClient(miniHome ...string) (libmachine.API, error) {
 		storePath:    storePath,
 		Filestore:    persist.NewFilestore(storePath, certsDir, certsDir),
 		legacyClient: NewRPCClient(storePath, certsDir),
+		flock:        fslock.New(localpath.MakeMiniPath("machine_client.lock")),
 	}, nil
 }
 
@@ -81,6 +83,7 @@ type LocalClient struct {
 	storePath string
 	*persist.Filestore
 	legacyClient libmachine.API
+	flock        *fslock.Lock
 }
 
 // NewHost creates a new Host
@@ -183,7 +186,22 @@ func (api *LocalClient) Create(h *host.Host) error {
 	}{
 		{
 			"bootstrapping certificates",
-			func() error { return cert.BootstrapCertificates(h.AuthOptions()) },
+			func() error {
+				// Lock is needed to avoid race conditiion in parallel Docker-Env test because issue #10107.
+				// CA cert and client cert should be generated atomically, otherwise might cause bad certificate error.
+				lockErr := api.flock.LockWithTimeout(time.Second * 5)
+				if lockErr != nil {
+					return fmt.Errorf("failed to acquire bootstrap client lock: %v " + lockErr.Error())
+				}
+				defer func() {
+					lockErr = api.flock.Unlock()
+					if lockErr != nil {
+						klog.Errorf("falied to release bootstrap cert client lock: %v", lockErr.Error())
+					}
+				}()
+				certErr := cert.BootstrapCertificates(h.AuthOptions())
+				return certErr
+			},
 		},
 		{
 			"precreate",