diff --git a/hack/test-end-to-end.sh b/hack/test-end-to-end.sh index 58582a51f1f5..f381f6ccfe4c 100755 --- a/hack/test-end-to-end.sh +++ b/hack/test-end-to-end.sh @@ -185,7 +185,9 @@ if [[ "${API_SCHEME}" == "https" ]]; then export CURL_KEY="${CERT_DIR}/admin/key.key" # Generate the certs first - wait_for_file "${CURL_CERT}" 0.5 80 + wait_for_file "${CERT_DIR}/openshift-client/key.key" 0.5 80 + wait_for_file "${CERT_DIR}/admin/key.key" 0.5 80 + wait_for_file "${CURL_CA_BUNDLE}" 0.5 80 # Read client cert data in to send to containerized components sudo chmod -R a+rX "${CERT_DIR}/openshift-client/" diff --git a/pkg/cmd/openshift/openshift.go b/pkg/cmd/openshift/openshift.go index 57a9d1f533c2..a60870b9de4d 100644 --- a/pkg/cmd/openshift/openshift.go +++ b/pkg/cmd/openshift/openshift.go @@ -80,7 +80,8 @@ func NewCommandOpenShift() *cobra.Command { root.SetUsageTemplate(templates.MainUsageTemplate()) root.SetHelpTemplate(templates.MainHelpTemplate()) - root.AddCommand(server.NewCommandStartServer("start")) + openshiftStartCommand, _ := server.NewCommandStartServer("start") + root.AddCommand(openshiftStartCommand) root.AddCommand(cli.NewCommandCLI("cli", "openshift cli")) root.AddCommand(cli.NewCmdKubectl("kube")) root.AddCommand(newExperimentalCommand("openshift", "ex")) diff --git a/pkg/cmd/server/command.go b/pkg/cmd/server/command.go new file mode 100644 index 000000000000..23bf810adf16 --- /dev/null +++ b/pkg/cmd/server/command.go @@ -0,0 +1,151 @@ +package server + +import ( + "errors" + "fmt" + _ "net/http/pprof" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + cmdutil "github.com/openshift/origin/pkg/cmd/util" +) + +const longCommandDesc = ` +Start an OpenShift server + +This command helps you launch an OpenShift server. The default mode is all-in-one, which allows +you to run all of the components of an OpenShift system on a server with Docker. Running + + $ openshift start + +will start OpenShift listening on all interfaces, launch an etcd server to store persistent +data, and launch the Kubernetes system components. The server will run in the foreground until +you terminate the process. + +Note: starting OpenShift without passing the --master address will attempt to find the IP +address that will be visible inside running Docker containers. This is not always successful, +so if you have problems tell OpenShift what public address it will be via --master=. + +You may also pass an optional argument to the start command to start OpenShift in one of the +following roles: + + $ openshift start master --nodes= + + Launches the server and control plane for OpenShift. You may pass a list of the node + hostnames you want to use, or create nodes via the REST API or 'openshift kube'. + + $ openshift start node --master= + + Launches a new node and attempts to connect to the master on the provided IP. + +You may also pass --etcd=
to connect to an external etcd server instead of running an +integrated instance, or --kubernetes= and --kubeconfig= to connect to an existing +Kubernetes cluster. +` + +// NewCommandStartServer provides a CLI handler for 'start' command +func NewCommandStartServer(name string) (*cobra.Command, *Config) { + cfg := NewDefaultConfig() + + cmd := &cobra.Command{ + Use: fmt.Sprintf("%s [master|node]", name), + Short: "Launch OpenShift", + Long: longCommandDesc, + Run: func(c *cobra.Command, args []string) { + if err := cfg.Validate(c.Flags().Args()); err != nil { + glog.Fatal(err) + } + + cfg.Complete(args) + + if err := start(*cfg, c.Flags().Args()); err != nil { + glog.Fatal(err) + } + }, + } + + flag := cmd.Flags() + + flag.BoolVar(&cfg.WriteConfigOnly, "write-config-and-walk-away", false, "Indicates that the command should write the config that would be used to start openshift and do nothing else. This is not yet implemented.") + + flag.Var(&cfg.BindAddr, "listen", "The address to listen for connections on (host, host:port, or URL).") + flag.Var(&cfg.MasterAddr, "master", "The master address for use by OpenShift components (host, host:port, or URL). Scheme and port default to the --listen scheme and port.") + flag.Var(&cfg.MasterPublicAddr, "public-master", "The master address for use by public clients, if different (host, host:port, or URL). Defaults to same as --master.") + flag.Var(&cfg.EtcdAddr, "etcd", "The address of the etcd server (host, host:port, or URL). If specified, no built-in etcd will be started.") + flag.Var(&cfg.KubernetesAddr, "kubernetes", "The address of the Kubernetes server (host, host:port, or URL). If specified, no Kubernetes components will be started.") + flag.Var(&cfg.KubernetesPublicAddr, "public-kubernetes", "The Kubernetes server address for use by public clients, if different. (host, host:port, or URL). Defaults to same as --kubernetes.") + flag.Var(&cfg.PortalNet, "portal-net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + + flag.StringVar(&cfg.ImageTemplate.Format, "images", cfg.ImageTemplate.Format, "When fetching images used by the cluster for important components, use this format on both master and nodes. The latest release will be used by default.") + flag.BoolVar(&cfg.ImageTemplate.Latest, "latest-images", cfg.ImageTemplate.Latest, "If true, attempt to use the latest images for the cluster instead of the latest release.") + + flag.StringVar(&cfg.VolumeDir, "volume-dir", "openshift.local.volumes", "The volume storage directory.") + flag.StringVar(&cfg.EtcdDir, "etcd-dir", "openshift.local.etcd", "The etcd data directory.") + flag.StringVar(&cfg.CertDir, "cert-dir", "openshift.local.certificates", "The certificate data directory.") + + flag.StringVar(&cfg.Hostname, "hostname", cfg.Hostname, "The hostname to identify this node with the master.") + flag.Var(&cfg.NodeList, "nodes", "The hostnames of each node. This currently must be specified up front. Comma delimited list") + flag.Var(&cfg.CORSAllowedOrigins, "cors-allowed-origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. CORS is enabled for localhost, 127.0.0.1, and the asset server by default.") + + cfg.ClientConfig = cmdutil.DefaultClientConfig(flag) + + cfg.Docker.InstallFlags(flag) + + return cmd, cfg +} + +const startMaster = "master" +const startNode = "node" + +func (cfg Config) Validate(args []string) error { + switch len(args) { + case 1: + switch args[0] { + case startMaster: // allowed case + case startNode: // allowed case + default: + return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") + } + case 0: + // do nothing, this starts an all in one + + default: + return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") + } + + return nil +} + +// Complete takes the args and fills in information for the start config +func (cfg *Config) Complete(args []string) { + cfg.ExplicitStartMaster = (len(args) == 1) && (args[0] == startMaster) + cfg.ExplicitStartNode = (len(args) == 1) && (args[0] == startNode) + + cfg.StartNode = true + cfg.StartMaster = true + cfg.StartKube = true + cfg.StartEtcd = true + + // if we're explicitly starting the master, don't start a node + if cfg.ExplicitStartMaster { + cfg.StartNode = false + } + + // if we're explicitly starting a node, don't start anything else + if cfg.ExplicitStartNode { + cfg.StartMaster = false + cfg.StartKube = false + cfg.StartEtcd = false + } + + // if we've explictly called out a kube location, don't start one in process + if cfg.KubernetesAddr.Provided { + cfg.StartKube = false + } + + // if we've explicitly called out an etcd location, don't start one in process + if cfg.EtcdAddr.Provided { + cfg.StartEtcd = false + } +} diff --git a/pkg/cmd/server/command_test.go b/pkg/cmd/server/command_test.go new file mode 100644 index 000000000000..de74b04c5f53 --- /dev/null +++ b/pkg/cmd/server/command_test.go @@ -0,0 +1,274 @@ +package server + +import ( + "strconv" + "testing" + + "github.com/spf13/cobra" +) + +func TestCommandBindingListen(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--listen=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.BindAddr.Set(valueToSet) + + if expectedConfig.BindAddr.String() != actualCfg.BindAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.BindAddr.String(), actualCfg.BindAddr.String()) + } +} + +func TestCommandBindingMaster(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--master=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.MasterAddr.Set(valueToSet) + + if expectedConfig.MasterAddr.String() != actualCfg.MasterAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.MasterAddr.String(), actualCfg.MasterAddr.String()) + } +} + +func TestCommandBindingMasterPublic(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--public-master=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.MasterPublicAddr.Set(valueToSet) + + if expectedConfig.MasterPublicAddr.String() != actualCfg.MasterPublicAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.MasterPublicAddr.String(), actualCfg.MasterPublicAddr.String()) + } +} + +func TestCommandBindingEtcd(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--etcd=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.EtcdAddr.Set(valueToSet) + + if expectedConfig.EtcdAddr.String() != actualCfg.EtcdAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.EtcdAddr.String(), actualCfg.EtcdAddr.String()) + } +} + +func TestCommandBindingKubernetes(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--kubernetes=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.KubernetesAddr.Set(valueToSet) + + if expectedConfig.KubernetesAddr.String() != actualCfg.KubernetesAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.KubernetesAddr.String(), actualCfg.KubernetesAddr.String()) + } +} + +func TestCommandBindingKubernetesPublic(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeCommand([]string{"--public-kubernetes=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.KubernetesPublicAddr.Set(valueToSet) + + if expectedConfig.KubernetesPublicAddr.String() != actualCfg.KubernetesPublicAddr.String() { + t.Errorf("expected %v, got %v", expectedConfig.KubernetesPublicAddr.String(), actualCfg.KubernetesPublicAddr.String()) + } +} + +func TestCommandBindingPortalNet(t *testing.T) { + valueToSet := "192.168.0.0/16" + actualCfg := executeCommand([]string{"--portal-net=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.PortalNet.Set(valueToSet) + + if expectedConfig.PortalNet.String() != actualCfg.PortalNet.String() { + t.Errorf("expected %v, got %v", expectedConfig.PortalNet.String(), actualCfg.PortalNet.String()) + } +} + +func TestCommandBindingImageTemplateFormat(t *testing.T) { + valueToSet := "some-format-string" + actualCfg := executeCommand([]string{"--images=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.ImageTemplate.Format = valueToSet + + if expectedConfig.ImageTemplate.Format != actualCfg.ImageTemplate.Format { + t.Errorf("expected %v, got %v", expectedConfig.ImageTemplate.Format, actualCfg.ImageTemplate.Format) + } +} + +func TestCommandBindingImageLatest(t *testing.T) { + expectedConfig := NewDefaultConfig() + + valueToSet := strconv.FormatBool(!expectedConfig.ImageTemplate.Latest) + actualCfg := executeCommand([]string{"--latest-images=" + valueToSet}) + + expectedConfig.ImageTemplate.Latest = !expectedConfig.ImageTemplate.Latest + + if expectedConfig.ImageTemplate.Latest != actualCfg.ImageTemplate.Latest { + t.Errorf("expected %v, got %v", expectedConfig.ImageTemplate.Latest, actualCfg.ImageTemplate.Latest) + } +} + +func TestCommandBindingVolumeDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeCommand([]string{"--volume-dir=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.VolumeDir = valueToSet + + if expectedConfig.VolumeDir != actualCfg.VolumeDir { + t.Errorf("expected %v, got %v", expectedConfig.VolumeDir, actualCfg.VolumeDir) + } +} + +func TestCommandBindingEtcdDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeCommand([]string{"--etcd-dir=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.EtcdDir = valueToSet + + if expectedConfig.EtcdDir != actualCfg.EtcdDir { + t.Errorf("expected %v, got %v", expectedConfig.EtcdDir, actualCfg.EtcdDir) + } +} + +func TestCommandBindingCertDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeCommand([]string{"--cert-dir=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.CertDir = valueToSet + + if expectedConfig.CertDir != actualCfg.CertDir { + t.Errorf("expected %v, got %v", expectedConfig.CertDir, actualCfg.CertDir) + } +} + +func TestCommandBindingHostname(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeCommand([]string{"--hostname=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.Hostname = valueToSet + + if expectedConfig.Hostname != actualCfg.Hostname { + t.Errorf("expected %v, got %v", expectedConfig.Hostname, actualCfg.Hostname) + } +} + +func TestCommandBindingNodes(t *testing.T) { + valueToSet := "first,second,third" + actualCfg := executeCommand([]string{"--nodes=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.NodeList.Set(valueToSet) + + if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) + } +} + +func TestCommandBindingCors(t *testing.T) { + valueToSet := "first,second,third" + actualCfg := executeCommand([]string{"--cors-allowed-origins=" + valueToSet}) + + expectedConfig := NewDefaultConfig() + expectedConfig.CORSAllowedOrigins.Set(valueToSet) + + if expectedConfig.CORSAllowedOrigins.String() != actualCfg.CORSAllowedOrigins.String() { + t.Errorf("expected %v, got %v", expectedConfig.CORSAllowedOrigins, actualCfg.CORSAllowedOrigins) + } +} + +func TestCommandCompletionNode(t *testing.T) { + commandCompletionTest{ + args: []string{"node"}, + + ExplicitStartNode: true, + StartNode: true, + }.run(t) +} +func TestCommandCompletionMaster(t *testing.T) { + commandCompletionTest{ + args: []string{"master"}, + + ExplicitStartMaster: true, + StartMaster: true, + StartKube: true, + StartEtcd: true, + }.run(t) +} +func TestCommandCompletionAllInOne(t *testing.T) { + commandCompletionTest{ + StartNode: true, + StartMaster: true, + StartKube: true, + StartEtcd: true, + }.run(t) +} + +type commandCompletionTest struct { + args []string + + ExplicitStartNode bool + ExplicitStartMaster bool + StartNode bool + StartMaster bool + StartKube bool + StartEtcd bool +} + +func executeCommand(args []string) *Config { + argsToUse := make([]string, 0, 1+len(args)) + argsToUse = append(argsToUse, "start") + argsToUse = append(argsToUse, args...) + argsToUse = append(argsToUse, "--write-config-and-walk-away") + + root := &cobra.Command{ + Use: "openshift", + Short: "test", + Long: "", + Run: func(c *cobra.Command, args []string) { + c.Help() + }, + } + + openshiftStartCommand, cfg := NewCommandStartServer("start") + root.AddCommand(openshiftStartCommand) + root.SetArgs(argsToUse) + root.Execute() + + return cfg +} + +func (test commandCompletionTest) run(t *testing.T) { + actualCfg := executeCommand(test.args) + + if test.ExplicitStartNode != actualCfg.ExplicitStartNode { + t.Errorf("expected %v, got %v", test.ExplicitStartNode, actualCfg.ExplicitStartNode) + } + if test.ExplicitStartMaster != actualCfg.ExplicitStartMaster { + t.Errorf("expected %v, got %v", test.ExplicitStartMaster, actualCfg.ExplicitStartNode) + } + if test.StartNode != actualCfg.StartNode { + t.Errorf("expected %v, got %v", test.StartNode, actualCfg.StartNode) + } + if test.StartMaster != actualCfg.StartMaster { + t.Errorf("expected %v, got %v", test.StartMaster, actualCfg.StartMaster) + } + if test.StartKube != actualCfg.StartKube { + t.Errorf("expected %v, got %v", test.StartKube, actualCfg.StartKube) + } + if test.StartEtcd != actualCfg.StartEtcd { + t.Errorf("expected %v, got %v", test.StartEtcd, actualCfg.StartEtcd) + } + +} diff --git a/pkg/cmd/server/config.go b/pkg/cmd/server/config.go new file mode 100644 index 000000000000..236bb0e5f0b9 --- /dev/null +++ b/pkg/cmd/server/config.go @@ -0,0 +1,279 @@ +package server + +import ( + "fmt" + "net" + _ "net/http/pprof" + "net/url" + "os/exec" + "strconv" + "strings" + "time" + + etcdclient "github.com/coreos/go-etcd/etcd" + "github.com/golang/glog" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + + "github.com/openshift/origin/pkg/api/latest" + "github.com/openshift/origin/pkg/cmd/flagtypes" + "github.com/openshift/origin/pkg/cmd/util" + "github.com/openshift/origin/pkg/cmd/util/docker" + "github.com/openshift/origin/pkg/cmd/util/variable" +) + +// config is a struct that the command stores flag values into. +type Config struct { + Docker *docker.Helper + + WriteConfigOnly bool + + ExplicitStartNode bool + ExplicitStartMaster bool + StartNode bool + StartMaster bool + StartKube bool + StartEtcd bool + + MasterAddr flagtypes.Addr + BindAddr flagtypes.Addr + EtcdAddr flagtypes.Addr + KubernetesAddr flagtypes.Addr + PortalNet flagtypes.IPNet + // addresses for external clients + MasterPublicAddr flagtypes.Addr + KubernetesPublicAddr flagtypes.Addr + + ImageFormat string + LatestReleaseImages bool + + ImageTemplate variable.ImageTemplate + + Hostname string + VolumeDir string + + EtcdDir string + + CertDir string + + StorageVersion string + + NodeList flagtypes.StringList + + // ClientConfig is used when connecting to Kubernetes from the master, or + // when connecting to the master from a detached node. If the server is an + // all-in-one, this value is not used. + ClientConfig clientcmd.ClientConfig + + CORSAllowedOrigins flagtypes.StringList +} + +func NewDefaultConfig() *Config { + hostname, err := defaultHostname() + if err != nil { + hostname = "localhost" + glog.Warningf("Unable to lookup hostname, using %q: %v", hostname, err) + } + + return &Config{ + Docker: docker.NewHelper(), + + MasterAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + BindAddr: flagtypes.Addr{Value: "0.0.0.0:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + EtcdAddr: flagtypes.Addr{Value: "0.0.0.0:4001", DefaultScheme: "http", DefaultPort: 4001}.Default(), + KubernetesAddr: flagtypes.Addr{DefaultScheme: "https", DefaultPort: 8443}.Default(), + PortalNet: flagtypes.DefaultIPNet("172.30.17.0/24"), + MasterPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + KubernetesPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + + ImageTemplate: variable.NewDefaultImageTemplate(), + + Hostname: hostname, + NodeList: flagtypes.StringList{"127.0.0.1"}, + } +} + +// GetMasterAddress checks for an unset master address and then attempts to use the first +// public IPv4 non-loopback address registered on this host. +// TODO: make me IPv6 safe +func (cfg Config) GetMasterAddress() (*url.URL, error) { + if cfg.MasterAddr.Provided { + return cfg.MasterAddr.URL, nil + } + + if cfg.StartMaster { + // If the user specifies a bind address, and the master is not provided, use the bind port by default + port := cfg.MasterAddr.Port + if cfg.BindAddr.Provided { + port = cfg.BindAddr.Port + } + + // If the user specifies a bind address, and the master is not provided, use the bind scheme by default + scheme := cfg.MasterAddr.URL.Scheme + if cfg.BindAddr.Provided { + scheme = cfg.BindAddr.URL.Scheme + } + + // use the default ip address for the system + addr, err := util.DefaultLocalIP4() + if err != nil { + return nil, fmt.Errorf("Unable to find the public address of this master: %v", err) + } + + masterAddr := scheme + "://" + net.JoinHostPort(addr.String(), strconv.Itoa(port)) + return url.Parse(masterAddr) + } + + // if we didn't specify and we aren't starting the master, read .kubeconfig to locate the master + // TODO client config currently doesn't let you override the defaults + // so it is defaulting to https://localhost:8443 for MasterAddr if + // it isn't set by --master or --kubeconfig + config, err := cfg.ClientConfig.ClientConfig() + if err != nil { + return nil, err + } + return url.Parse(config.Host) +} + +func (cfg Config) GetMasterPublicAddress() (*url.URL, error) { + if cfg.MasterPublicAddr.Provided { + return cfg.MasterPublicAddr.URL, nil + } + + return cfg.GetMasterAddress() +} + +func (cfg Config) GetEtcdBindAddress() string { + // Derive the etcd bind address by using the bind address and the default etcd port + return net.JoinHostPort(cfg.BindAddr.Host, strconv.Itoa(cfg.EtcdAddr.DefaultPort)) +} + +func (cfg Config) GetEtcdPeerBindAddress() string { + // Derive the etcd peer address by using the bind address and the default etcd peering port + return net.JoinHostPort(cfg.BindAddr.Host, "7001") +} + +func (cfg Config) GetEtcdAddress() (*url.URL, error) { + if cfg.EtcdAddr.Provided { + return cfg.EtcdAddr.URL, nil + } + + // Etcd should be reachable on the same address that the master is (for simplicity) + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + + etcdAddr := net.JoinHostPort(getHost(*masterAddr), strconv.Itoa(cfg.EtcdAddr.DefaultPort)) + return url.Parse("http://" + etcdAddr) +} + +func (cfg Config) GetKubernetesAddress() (*url.URL, error) { + if cfg.KubernetesAddr.Provided { + return cfg.KubernetesAddr.URL, nil + } + + return cfg.GetMasterAddress() +} + +func (cfg Config) GetKubernetesPublicAddress() (*url.URL, error) { + if cfg.KubernetesPublicAddr.Provided { + return cfg.KubernetesPublicAddr.URL, nil + } + if cfg.KubernetesAddr.Provided { + return cfg.KubernetesAddr.URL, nil + } + + return cfg.GetMasterPublicAddress() +} + +func (cfg Config) GetAssetPublicAddress() (*url.URL, error) { + // Derive the asset public address by incrementing the master public address port by 1 + t, err := cfg.GetMasterPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr := *t + assetPublicAddr.Host = net.JoinHostPort(getHost(assetPublicAddr), strconv.Itoa(getPort(assetPublicAddr)+1)) + + return &assetPublicAddr, nil +} + +func (cfg Config) GetAssetBindAddress() string { + // Derive the asset bind address by incrementing the master bind address port by 1 + return net.JoinHostPort(cfg.BindAddr.Host, strconv.Itoa(cfg.BindAddr.Port+1)) +} + +func (cfg Config) GetNodeList() []string { + nodeList := []string{} + for _, curr := range cfg.NodeList { + nodeList = append(nodeList, curr) + } + + if len(nodeList) == 1 && nodeList[0] == "127.0.0.1" { + nodeList[0] = cfg.Hostname + } + for i, s := range nodeList { + s = strings.ToLower(s) + nodeList[i] = s + } + + return nodeList +} + +// getEtcdClient creates an etcd client based on the provided config and waits +// until etcd server is reachable. It errors out and exits if the server cannot +// be reached for a certain amount of time. +func (cfg Config) getEtcdClient() (*etcdclient.Client, error) { + address, err := cfg.GetEtcdAddress() + if err != nil { + return nil, err + } + etcdServers := []string{address.String()} + etcdClient := etcdclient.NewClient(etcdServers) + + for i := 0; ; i++ { + _, err := etcdClient.Get("/", false, false) + if err == nil || tools.IsEtcdNotFound(err) { + break + } + if i > 100 { + return nil, fmt.Errorf("Could not reach etcd: %v", err) + } + time.Sleep(50 * time.Millisecond) + } + + return etcdClient, nil +} + +// newOpenShiftEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version +// is incorrect. +func (cfg Config) newOpenShiftEtcdHelper() (helper tools.EtcdHelper, err error) { + // Connect and setup etcd interfaces + client, err := cfg.getEtcdClient() + if err != nil { + return tools.EtcdHelper{}, err + } + + version := cfg.StorageVersion + if len(version) == 0 { + version = latest.Version + } + interfaces, err := latest.InterfacesFor(version) + if err != nil { + return helper, err + } + return tools.EtcdHelper{client, interfaces.Codec, tools.RuntimeVersionAdapter{interfaces.MetadataAccessor}}, nil +} + +// defaultHostname returns the default hostname for this system. +func defaultHostname() (string, error) { + // Note: We use exec here instead of os.Hostname() because we + // want the FQDN, and this is the easiest way to get it. + fqdn, err := exec.Command("hostname", "-f").Output() + if err != nil { + return "", fmt.Errorf("Couldn't determine hostname: %v", err) + } + return strings.TrimSpace(string(fqdn)), nil +} diff --git a/pkg/cmd/server/config_test.go b/pkg/cmd/server/config_test.go new file mode 100644 index 000000000000..4a830fae4627 --- /dev/null +++ b/pkg/cmd/server/config_test.go @@ -0,0 +1,234 @@ +package server + +import ( + "testing" + + "github.com/openshift/origin/pkg/cmd/util" +) + +func TestMasterPublicAddressDefaulting(t *testing.T) { + expected := "http://example.com:9012" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set(expected) + + actual, err := cfg.GetMasterPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestMasterPublicAddressExplicit(t *testing.T) { + expected := "http://external.com:12445" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set("http://internal.com:9012") + cfg.MasterPublicAddr.Set(expected) + + actual, err := cfg.GetMasterPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesPublicAddressDefaultToKubernetesAddress(t *testing.T) { + expected := "http://example.com:9012" + + cfg := NewDefaultConfig() + cfg.KubernetesAddr.Set(expected) + cfg.MasterPublicAddr.Set("unexpectedpublicmaster") + cfg.MasterAddr.Set("unexpectedmaster") + + actual, err := cfg.GetKubernetesPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesPublicAddressDefaultToPublicMasterAddress(t *testing.T) { + expected := "http://example.com:9012" + + cfg := NewDefaultConfig() + cfg.MasterPublicAddr.Set(expected) + cfg.MasterAddr.Set("unexpectedmaster") + + actual, err := cfg.GetKubernetesPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesPublicAddressDefaultToMasterAddress(t *testing.T) { + expected := "http://example.com:9012" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set(expected) + + actual, err := cfg.GetKubernetesPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesPublicAddressExplicit(t *testing.T) { + expected := "http://external.com:12445" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set("http://internal.com:9012") + cfg.KubernetesAddr.Set("http://internal.com:9013") + cfg.MasterPublicAddr.Set("http://internal.com:9014") + cfg.KubernetesPublicAddr.Set(expected) + + actual, err := cfg.GetKubernetesPublicAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesAddressDefaulting(t *testing.T) { + expected := "http://example.com:9012" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set(expected) + + actual, err := cfg.GetKubernetesAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestKubernetesAddressExplicit(t *testing.T) { + expected := "http://external.com:12445" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set("http://internal.com:9012") + cfg.KubernetesAddr.Set(expected) + + actual, err := cfg.GetKubernetesAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestEtcdAddressDefaulting(t *testing.T) { + expected := "http://example.com:4001" + master := "https://example.com:9012" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set(master) + + actual, err := cfg.GetEtcdAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestEtcdAddressExplicit(t *testing.T) { + expected := "http://external.com:12445" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set("http://internal.com:9012") + cfg.EtcdAddr.Set(expected) + + actual, err := cfg.GetEtcdAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestEtcdBindAddressDefault(t *testing.T) { + expected := "0.0.0.0:4001" + + cfg := NewDefaultConfig() + actual := cfg.GetEtcdBindAddress() + if expected != actual { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestEtcdPeerAddressDefault(t *testing.T) { + expected := "0.0.0.0:7001" + + cfg := NewDefaultConfig() + actual := cfg.GetEtcdPeerBindAddress() + if expected != actual { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestEtcdBindAddressDefaultToBind(t *testing.T) { + expected := "1.2.3.4:4001" + + cfg := NewDefaultConfig() + cfg.BindAddr.Set("https://1.2.3.4:8080") + + actual := cfg.GetEtcdBindAddress() + if expected != actual { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestMasterAddressDefaultingToBindValues(t *testing.T) { + defaultIP, err := util.DefaultLocalIP4() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + expected := "http://" + defaultIP.String() + ":9012" + + cfg := NewDefaultConfig() + cfg.StartMaster = true + cfg.BindAddr.Set("http://0.0.0.0:9012") + + actual, err := cfg.GetMasterAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} + +func TestMasterAddressExplicit(t *testing.T) { + expected := "http://external.com:12445" + + cfg := NewDefaultConfig() + cfg.MasterAddr.Set(expected) + + actual, err := cfg.GetMasterAddress() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if expected != actual.String() { + t.Errorf("expected %v, got %v", expected, actual) + } +} diff --git a/pkg/cmd/server/kube_master.go b/pkg/cmd/server/kube_master.go new file mode 100644 index 000000000000..853748c477bd --- /dev/null +++ b/pkg/cmd/server/kube_master.go @@ -0,0 +1,60 @@ +package server + +import ( + "fmt" + "net" + _ "net/http/pprof" + + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + kmaster "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" + + "github.com/openshift/origin/pkg/cmd/server/kubernetes" +) + +func (cfg Config) BuildKubernetesMasterConfig(requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*kubernetes.MasterConfig, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + + // Connect and setup etcd interfaces + etcdClient, err := cfg.getEtcdClient() + if err != nil { + return nil, err + } + ketcdHelper, err := kmaster.NewEtcdHelper(etcdClient, klatest.Version) + if err != nil { + return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err) + } + + portalNet := net.IPNet(cfg.PortalNet) + masterIP := net.ParseIP(getHost(*masterAddr)) + if masterIP == nil { + addrs, err := net.LookupIP(getHost(*masterAddr)) + if err != nil { + return nil, fmt.Errorf("Unable to find an IP for %q - specify an IP directly? %v", getHost(*masterAddr), err) + } + if len(addrs) == 0 { + return nil, fmt.Errorf("Unable to find an IP for %q - specify an IP directly?", getHost(*masterAddr)) + } + masterIP = addrs[0] + } + + kmaster := &kubernetes.MasterConfig{ + MasterIP: masterIP, + MasterPort: cfg.MasterAddr.Port, + NodeHosts: cfg.NodeList, + PortalNet: &portalNet, + RequestContextMapper: requestContextMapper, + EtcdHelper: ketcdHelper, + KubeClient: kubeClient, + Authorizer: apiserver.NewAlwaysAllowAuthorizer(), + AdmissionControl: admit.NewAlwaysAdmit(), + } + + return kmaster, nil +} diff --git a/pkg/cmd/server/node_config.go b/pkg/cmd/server/node_config.go new file mode 100644 index 000000000000..feaf90560fd8 --- /dev/null +++ b/pkg/cmd/server/node_config.go @@ -0,0 +1,37 @@ +package server + +import ( + _ "net/http/pprof" + + "github.com/openshift/origin/pkg/cmd/server/kubernetes" +) + +func (cfg Config) BuildKubernetesNodeConfig() (*kubernetes.NodeConfig, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + kubeClient, _, err := cfg.GetKubeClient() + if err != nil { + return nil, err + } + + // define a function for resolving components to names + imageResolverFn := cfg.ImageTemplate.ExpandOrDie + + nodeConfig := &kubernetes.NodeConfig{ + BindHost: cfg.BindAddr.Host, + NodeHost: cfg.Hostname, + MasterHost: masterAddr.String(), + + VolumeDir: cfg.VolumeDir, + + NetworkContainerImage: imageResolverFn("pod"), + + AllowDisabledDocker: cfg.StartNode && cfg.StartMaster, + + Client: kubeClient, + } + + return nodeConfig, nil +} diff --git a/pkg/cmd/server/origin/config.go b/pkg/cmd/server/origin/config.go new file mode 100644 index 000000000000..f04493160170 --- /dev/null +++ b/pkg/cmd/server/origin/config.go @@ -0,0 +1,208 @@ +package origin + +import ( + "crypto/x509" + "net/http" + "strings" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + kutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" + + "github.com/openshift/origin/pkg/api/latest" + "github.com/openshift/origin/pkg/auth/authenticator" + "github.com/openshift/origin/pkg/auth/authenticator/request/bearertoken" + "github.com/openshift/origin/pkg/auth/authenticator/request/paramtoken" + "github.com/openshift/origin/pkg/auth/authenticator/request/unionrequest" + "github.com/openshift/origin/pkg/auth/authenticator/request/x509request" + "github.com/openshift/origin/pkg/auth/group" + authnregistry "github.com/openshift/origin/pkg/auth/oauth/registry" + "github.com/openshift/origin/pkg/authorization/authorizer" + policycache "github.com/openshift/origin/pkg/authorization/cache" + authorizationetcd "github.com/openshift/origin/pkg/authorization/registry/etcd" + "github.com/openshift/origin/pkg/authorization/rulevalidation" + osclient "github.com/openshift/origin/pkg/client" + "github.com/openshift/origin/pkg/cmd/util/variable" + oauthetcd "github.com/openshift/origin/pkg/oauth/registry/etcd" + projectauth "github.com/openshift/origin/pkg/project/auth" +) + +const ( + unauthenticatedUsername = "system:anonymous" + + authenticatedGroup = "system:authenticated" + unauthenticatedGroup = "system:unauthenticated" +) + +type MasterConfigParameters struct { + // host:port to bind master to + MasterBindAddr string + // host:port to bind asset server to + AssetBindAddr string + // url to access the master API on within the cluster + MasterAddr string + // url to access kubernetes API on within the cluster + KubernetesAddr string + // external clients may need to access APIs at different addresses than internal components do + MasterPublicAddr string + KubernetesPublicAddr string + AssetPublicAddr string + // LogoutURI is an optional, absolute URI to redirect web browsers to after logging out of the web console. + // If not specified, the built-in logout page is shown. + LogoutURI string + + CORSAllowedOrigins []string + + EtcdHelper tools.EtcdHelper + + MasterCertFile string + MasterKeyFile string + AssetCertFile string + AssetKeyFile string + + // ClientCAs will be used to request client certificates in connections to the API. + // This CertPool should contain all the CAs that will be used for client certificate verification. + ClientCAs *x509.CertPool + + MasterAuthorizationNamespace string + + // kubeClient is the client used to call Kubernetes APIs from system components, built from KubeClientConfig. + // It should only be accessed via the *Client() helper methods. + // To apply different access control to a system component, create a separate client/config specifically for that component. + KubeClient *kclient.Client + // KubeClientConfig is the client configuration used to call Kubernetes APIs from system components. + // To apply different access control to a system component, create a client config specifically for that component. + KubeClientConfig kclient.Config + + // osClient is the client used to call OpenShift APIs from system components, built from OSClientConfig. + // It should only be accessed via the *Client() helper methods. + // To apply different access control to a system component, create a separate client/config specifically for that component. + OSClient *osclient.Client + // OSClientConfig is the client configuration used to call OpenShift APIs from system components + // To apply different access control to a system component, create a client config specifically for that component. + OSClientConfig kclient.Config + + // DeployerOSClientConfig is the client configuration used to call OpenShift APIs from launched deployer pods + DeployerOSClientConfig kclient.Config +} + +// MasterConfig defines the required parameters for starting the OpenShift master +type MasterConfig struct { + MasterConfigParameters + + Authenticator authenticator.Request + Authorizer authorizer.Authorizer + AuthorizationAttributeBuilder authorizer.AuthorizationAttributeBuilder + + PolicyCache *policycache.PolicyCache + ProjectAuthorizationCache *projectauth.AuthorizationCache + + // Map requests to contexts + RequestContextMapper kapi.RequestContextMapper + + AdmissionControl admission.Interface + + // a function that returns the appropriate image to use for a named component + ImageFor func(component string) string + + TLS bool +} + +func BuildMasterConfig(configParams MasterConfigParameters) (*MasterConfig, error) { + + policyCache := configParams.newPolicyCache() + requestContextMapper := kapi.NewRequestContextMapper() + + imageTemplate := variable.NewDefaultImageTemplate() + + config := &MasterConfig{ + MasterConfigParameters: configParams, + + Authenticator: configParams.newAuthenticator(), + Authorizer: newAuthorizer(policyCache, configParams.MasterAuthorizationNamespace), + AuthorizationAttributeBuilder: newAuthorizationAttributeBuilder(requestContextMapper), + + PolicyCache: policyCache, + ProjectAuthorizationCache: configParams.newProjectAuthorizationCache(), + + RequestContextMapper: requestContextMapper, + + AdmissionControl: admit.NewAlwaysAdmit(), + + ImageFor: imageTemplate.ExpandOrDie, + + TLS: strings.HasPrefix(configParams.MasterAddr, "https://"), + } + + return config, nil +} + +func (c MasterConfigParameters) newAuthenticator() authenticator.Request { + useTLS := strings.HasPrefix(c.MasterAddr, "https://") + + tokenAuthenticator := getEtcdTokenAuthenticator(c.EtcdHelper) + + authenticators := []authenticator.Request{} + authenticators = append(authenticators, bearertoken.New(tokenAuthenticator, true)) + // Allow token as access_token param for WebSockets + // TODO: make the param name configurable + // TODO: limit this authenticator to watch methods, if possible + // TODO: prevent access_token param from getting logged, if possible + authenticators = append(authenticators, paramtoken.New("access_token", tokenAuthenticator, true)) + + if useTLS { + // build cert authenticator + // TODO: add cert users to etcd? + opts := x509request.DefaultVerifyOptions() + opts.Roots = c.ClientCAs + certauth := x509request.New(opts, x509request.SubjectToUserConversion) + authenticators = append(authenticators, certauth) + } + + // TODO: make anonymous auth optional? + ret := &unionrequest.Authenticator{ + FailOnError: true, + Handlers: []authenticator.Request{ + group.NewGroupAdder(unionrequest.NewUnionAuthentication(authenticators...), []string{authenticatedGroup}), + authenticator.RequestFunc(func(req *http.Request) (user.Info, bool, error) { + return &user.DefaultInfo{Name: unauthenticatedUsername, Groups: []string{unauthenticatedGroup}}, true, nil + }), + }, + } + + return ret +} + +func (c MasterConfigParameters) newProjectAuthorizationCache() *projectauth.AuthorizationCache { + return projectauth.NewAuthorizationCache( + projectauth.NewReviewer(c.OSClient), + c.KubeClient.Namespaces(), + c.OSClient, + c.OSClient, + c.MasterAuthorizationNamespace) +} + +func (c MasterConfigParameters) newPolicyCache() *policycache.PolicyCache { + authorizationEtcd := authorizationetcd.New(c.EtcdHelper) + return policycache.NewPolicyCache(authorizationEtcd, authorizationEtcd) +} + +func newAuthorizer(policyCache *policycache.PolicyCache, masterAuthorizationNamespace string) authorizer.Authorizer { + authorizer := authorizer.NewAuthorizer(masterAuthorizationNamespace, rulevalidation.NewDefaultRuleResolver(policyCache, policyCache)) + return authorizer +} + +func newAuthorizationAttributeBuilder(requestContextMapper kapi.RequestContextMapper) authorizer.AuthorizationAttributeBuilder { + authorizationAttributeBuilder := authorizer.NewAuthorizationAttributeBuilder(requestContextMapper, &apiserver.APIRequestInfoResolver{kutil.NewStringSet("api", "osapi"), latest.RESTMapper}) + return authorizationAttributeBuilder +} + +func getEtcdTokenAuthenticator(etcdHelper tools.EtcdHelper) authenticator.Token { + oauthRegistry := oauthetcd.New(etcdHelper) + return authnregistry.NewTokenAuthenticator(oauthRegistry) +} diff --git a/pkg/cmd/server/origin/master.go b/pkg/cmd/server/origin/master.go index 7b17e0c182a0..dcac9b0677db 100644 --- a/pkg/cmd/server/origin/master.go +++ b/pkg/cmd/server/origin/master.go @@ -2,7 +2,6 @@ package origin import ( "crypto/tls" - "crypto/x509" "fmt" "net/http" "net/url" @@ -17,7 +16,6 @@ import ( "github.com/emicklei/go-restful/swagger" "github.com/golang/glog" - "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" @@ -30,7 +28,6 @@ import ( "github.com/openshift/origin/pkg/api/latest" "github.com/openshift/origin/pkg/api/v1beta1" "github.com/openshift/origin/pkg/assets" - "github.com/openshift/origin/pkg/auth/authenticator" buildclient "github.com/openshift/origin/pkg/build/client" buildcontrollerfactory "github.com/openshift/origin/pkg/build/controller/factory" buildstrategy "github.com/openshift/origin/pkg/build/controller/strategy" @@ -60,7 +57,6 @@ import ( clientregistry "github.com/openshift/origin/pkg/oauth/registry/client" clientauthorizationregistry "github.com/openshift/origin/pkg/oauth/registry/clientauthorization" oauthetcd "github.com/openshift/origin/pkg/oauth/registry/etcd" - projectauth "github.com/openshift/origin/pkg/project/auth" projectregistry "github.com/openshift/origin/pkg/project/registry/project" routeetcd "github.com/openshift/origin/pkg/route/registry/etcd" routeregistry "github.com/openshift/origin/pkg/route/registry/route" @@ -75,7 +71,6 @@ import ( authorizationapi "github.com/openshift/origin/pkg/authorization/api" "github.com/openshift/origin/pkg/authorization/authorizer" - policycache "github.com/openshift/origin/pkg/authorization/cache" authorizationetcd "github.com/openshift/origin/pkg/authorization/registry/etcd" policyregistry "github.com/openshift/origin/pkg/authorization/registry/policy" policybindingregistry "github.com/openshift/origin/pkg/authorization/registry/policybinding" @@ -92,74 +87,6 @@ const ( swaggerAPIPrefix = "/swaggerapi/" ) -// MasterConfig defines the required parameters for starting the OpenShift master -type MasterConfig struct { - // host:port to bind master to - MasterBindAddr string - // host:port to bind asset server to - AssetBindAddr string - // url to access the master API on within the cluster - MasterAddr string - // url to access kubernetes API on within the cluster - KubernetesAddr string - // external clients may need to access APIs at different addresses than internal components do - MasterPublicAddr string - KubernetesPublicAddr string - AssetPublicAddr string - // LogoutURI is an optional, absolute URI to redirect web browsers to after logging out of the web console. - // If not specified, the built-in logout page is shown. - LogoutURI string - - CORSAllowedOrigins []string - Authenticator authenticator.Request - Authorizer authorizer.Authorizer - AuthorizationAttributeBuilder authorizer.AuthorizationAttributeBuilder - MasterAuthorizationNamespace string - - PolicyCache *policycache.PolicyCache - ProjectAuthorizationCache *projectauth.AuthorizationCache - - // Map requests to contexts - RequestContextMapper kapi.RequestContextMapper - - EtcdHelper tools.EtcdHelper - - AdmissionControl admission.Interface - - // a function that returns the appropriate image to use for a named component - ImageFor func(component string) string - - TLS bool - - MasterCertFile string - MasterKeyFile string - AssetCertFile string - AssetKeyFile string - - // ClientCAs will be used to request client certificates in connections to the API. - // This CertPool should contain all the CAs that will be used for client certificate verification. - ClientCAs *x509.CertPool - - // kubeClient is the client used to call Kubernetes APIs from system components, built from KubeClientConfig. - // It should only be accessed via the *Client() helper methods. - // To apply different access control to a system component, create a separate client/config specifically for that component. - kubeClient *kclient.Client - // KubeClientConfig is the client configuration used to call Kubernetes APIs from system components. - // To apply different access control to a system component, create a client config specifically for that component. - KubeClientConfig kclient.Config - - // osClient is the client used to call OpenShift APIs from system components, built from OSClientConfig. - // It should only be accessed via the *Client() helper methods. - // To apply different access control to a system component, create a separate client/config specifically for that component. - osClient *osclient.Client - // OSClientConfig is the client configuration used to call OpenShift APIs from system components - // To apply different access control to a system component, create a client config specifically for that component. - OSClientConfig kclient.Config - - // DeployerOSClientConfig is the client configuration used to call OpenShift APIs from launched deployer pods - DeployerOSClientConfig kclient.Config -} - // APIInstaller installs additional API components into this server type APIInstaller interface { // Returns an array of strings describing what was installed @@ -174,23 +101,9 @@ func (fn APIInstallFunc) InstallAPI(container *restful.Container) []string { return fn(container) } -func (c *MasterConfig) BuildClients() { - kubeClient, err := kclient.New(&c.KubeClientConfig) - if err != nil { - glog.Fatalf("Unable to configure client: %v", err) - } - c.kubeClient = kubeClient - - osclient, err := osclient.New(&c.OSClientConfig) - if err != nil { - glog.Fatalf("Unable to configure client: %v", err) - } - c.osClient = osclient -} - // KubeClient returns the kubernetes client object func (c *MasterConfig) KubeClient() *kclient.Client { - return c.kubeClient + return c.MasterConfigParameters.KubeClient } // PolicyClient returns the policy client object @@ -199,37 +112,37 @@ func (c *MasterConfig) KubeClient() *kclient.Client { // list, watch all policies in all namespaces // create resourceAccessReviews in all namespaces func (c *MasterConfig) PolicyClient() *osclient.Client { - return c.osClient + return c.OSClient } // DeploymentClient returns the deployment client object func (c *MasterConfig) DeploymentClient() *kclient.Client { - return c.kubeClient + return c.MasterConfigParameters.KubeClient } // BuildLogClient returns the build log client object func (c *MasterConfig) BuildLogClient() *kclient.Client { - return c.kubeClient + return c.MasterConfigParameters.KubeClient } // WebHookClient returns the webhook client object func (c *MasterConfig) WebHookClient() *osclient.Client { - return c.osClient + return c.OSClient } // BuildControllerClients returns the build controller client objects func (c *MasterConfig) BuildControllerClients() (*osclient.Client, *kclient.Client) { - return c.osClient, c.kubeClient + return c.OSClient, c.MasterConfigParameters.KubeClient } // ImageChangeControllerClient returns the openshift client object func (c *MasterConfig) ImageChangeControllerClient() *osclient.Client { - return c.osClient + return c.OSClient } // DeploymentControllerClients returns the deployment controller client object func (c *MasterConfig) DeploymentControllerClients() (*osclient.Client, *kclient.Client) { - return c.osClient, c.kubeClient + return c.OSClient, c.MasterConfigParameters.KubeClient } // DeployerClientConfig returns the client configuration a Deployer instance launched in a pod @@ -239,13 +152,13 @@ func (c *MasterConfig) DeployerClientConfig() *kclient.Config { } func (c *MasterConfig) DeploymentConfigControllerClients() (*osclient.Client, *kclient.Client) { - return c.osClient, c.kubeClient + return c.OSClient, c.MasterConfigParameters.KubeClient } func (c *MasterConfig) DeploymentConfigChangeControllerClients() (*osclient.Client, *kclient.Client) { - return c.osClient, c.kubeClient + return c.OSClient, c.MasterConfigParameters.KubeClient } func (c *MasterConfig) DeploymentImageChangeControllerClient() *osclient.Client { - return c.osClient + return c.OSClient } func (c *MasterConfig) InstallProtectedAPI(container *restful.Container) []string { diff --git a/pkg/cmd/server/origin_master.go b/pkg/cmd/server/origin_master.go new file mode 100644 index 000000000000..84b02c54c915 --- /dev/null +++ b/pkg/cmd/server/origin_master.go @@ -0,0 +1,416 @@ +package server + +import ( + "crypto/x509" + "errors" + "fmt" + "net" + _ "net/http/pprof" + "strings" + "time" + + "code.google.com/p/go-uuid/uuid" + + klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + osclient "github.com/openshift/origin/pkg/client" + "github.com/openshift/origin/pkg/cmd/server/crypto" + "github.com/openshift/origin/pkg/cmd/server/origin" +) + +func (cfg Config) BuildOriginMasterConfig() (*origin.MasterConfig, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + kubeAddr, err := cfg.GetKubernetesAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := cfg.GetMasterPublicAddress() + if err != nil { + return nil, err + } + kubePublicAddr, err := cfg.GetKubernetesPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr, err := cfg.GetAssetPublicAddress() + if err != nil { + return nil, err + } + + corsAllowedOrigins := []string{} + corsAllowedOrigins = append(corsAllowedOrigins, cfg.CORSAllowedOrigins...) + // always include the all-in-one server's web console as an allowed CORS origin + // always include localhost as an allowed CORS origin + // always include master public address as an allowed CORS origin + for _, origin := range []string{assetPublicAddr.Host, masterPublicAddr.Host, "localhost", "127.0.0.1"} { + // TODO: check if origin is already allowed + corsAllowedOrigins = append(corsAllowedOrigins, origin) + } + + etcdHelper, err := cfg.newOpenShiftEtcdHelper() + if err != nil { + return nil, fmt.Errorf("Error setting up server storage: %v", err) + } + + masterCertFile, masterKeyFile, err := cfg.GetMasterCert() + if err != nil { + return nil, err + } + assetCertFile, assetKeyFile, err := cfg.GetAssetCert() + if err != nil { + return nil, err + } + + clientCAs, err := cfg.GetClientCAs() + if err != nil { + return nil, err + } + + kubeClient, kubeClientConfig, err := cfg.GetKubeClient() + openshiftClient, openshiftClientConfig, err := cfg.GetOpenshiftClient() + deployerClientConfig, err := cfg.GetOpenshiftDeployerClientConfig() + + openshiftConfigParameters := origin.MasterConfigParameters{ + MasterBindAddr: cfg.BindAddr.URL.Host, + AssetBindAddr: cfg.GetAssetBindAddress(), + MasterAddr: masterAddr.String(), + KubernetesAddr: kubeAddr.String(), + MasterPublicAddr: masterPublicAddr.String(), + KubernetesPublicAddr: kubePublicAddr.String(), + AssetPublicAddr: assetPublicAddr.String(), + + CORSAllowedOrigins: corsAllowedOrigins, + MasterAuthorizationNamespace: "master", + LogoutURI: env("OPENSHIFT_LOGOUT_URI", ""), + + EtcdHelper: etcdHelper, + + MasterCertFile: masterCertFile, + MasterKeyFile: masterKeyFile, + AssetCertFile: assetCertFile, + AssetKeyFile: assetKeyFile, + ClientCAs: clientCAs, + + KubeClient: kubeClient, + KubeClientConfig: *kubeClientConfig, + OSClient: openshiftClient, + OSClientConfig: *openshiftClientConfig, + DeployerOSClientConfig: *deployerClientConfig, + } + openshiftConfig, err := origin.BuildMasterConfig(openshiftConfigParameters) + if err != nil { + return nil, err + } + + return openshiftConfig, nil +} + +func (cfg Config) BuildAuthConfig() (*origin.AuthConfig, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := cfg.GetMasterPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr, err := cfg.GetAssetPublicAddress() + if err != nil { + return nil, err + } + + clientCAs, err := cfg.GetClientCAs() + if err != nil { + return nil, err + } + + // Build the list of valid redirect_uri prefixes for a login using the openshift-web-console client to redirect to + // TODO: allow configuring this + // TODO: remove hard-coding of development UI server + assetPublicAddresses := []string{assetPublicAddr.String(), "http://localhost:9000", "https://localhost:9000"} + + etcdHelper, err := cfg.newOpenShiftEtcdHelper() + if err != nil { + return nil, fmt.Errorf("Error setting up server storage: %v", err) + } + // Default to a session authenticator (for browsers), and a basicauth authenticator (for clients responding to WWW-Authenticate challenges) + defaultAuthRequestHandlers := strings.Join([]string{ + string(origin.AuthRequestHandlerSession), + string(origin.AuthRequestHandlerBasicAuth), + }, ",") + + ret := &origin.AuthConfig{ + MasterAddr: masterAddr.String(), + MasterPublicAddr: masterPublicAddr.String(), + AssetPublicAddresses: assetPublicAddresses, + MasterRoots: clientCAs, + EtcdHelper: etcdHelper, + + // Max token ages + AuthorizeTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_AUTHORIZE_TOKEN_MAX_AGE_SECONDS", 300, 1), + AccessTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_ACCESS_TOKEN_MAX_AGE_SECONDS", 3600, 1), + // Handlers + AuthRequestHandlers: origin.ParseAuthRequestHandlerTypes(env("OPENSHIFT_OAUTH_REQUEST_HANDLERS", defaultAuthRequestHandlers)), + AuthHandler: origin.AuthHandlerType(env("OPENSHIFT_OAUTH_HANDLER", string(origin.AuthHandlerLogin))), + GrantHandler: origin.GrantHandlerType(env("OPENSHIFT_OAUTH_GRANT_HANDLER", string(origin.GrantHandlerAuto))), + // RequestHeader config + RequestHeaders: strings.Split(env("OPENSHIFT_OAUTH_REQUEST_HEADERS", "X-Remote-User"), ","), + // Session config (default to unknowable secret) + SessionSecrets: []string{env("OPENSHIFT_OAUTH_SESSION_SECRET", uuid.NewUUID().String())}, + SessionMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_SESSION_MAX_AGE_SECONDS", 300, 1), + SessionName: env("OPENSHIFT_OAUTH_SESSION_NAME", "ssn"), + // Password config + PasswordAuth: origin.PasswordAuthType(env("OPENSHIFT_OAUTH_PASSWORD_AUTH", string(origin.PasswordAuthAnyPassword))), + BasicAuthURL: env("OPENSHIFT_OAUTH_BASIC_AUTH_URL", ""), + // Token config + TokenStore: origin.TokenStoreType(env("OPENSHIFT_OAUTH_TOKEN_STORE", string(origin.TokenStoreOAuth))), + TokenFilePath: env("OPENSHIFT_OAUTH_TOKEN_FILE_PATH", ""), + // Google config + GoogleClientID: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_ID", ""), + GoogleClientSecret: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_SECRET", ""), + // GitHub config + GithubClientID: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_ID", ""), + GithubClientSecret: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_SECRET", ""), + } + + return ret, nil + +} + +func (cfg Config) newCA() (*crypto.CA, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + + // Bootstrap CA + // TODO: store this (or parts of this) in etcd? + ca, err := crypto.InitCA(cfg.CertDir, fmt.Sprintf("%s@%d", masterAddr.Host, time.Now().Unix())) + if err != nil { + return nil, fmt.Errorf("Unable to configure certificate authority: %v", err) + } + + return ca, nil +} + +func (cfg Config) GetClientCAs() (*x509.CertPool, error) { + ca, err := cfg.newCA() + if err != nil { + return nil, err + } + + // Save cert roots + roots := x509.NewCertPool() + for _, root := range ca.Config.Roots { + roots.AddCert(root) + } + + return roots, nil +} + +func (cfg Config) GetServerCertHostnames() ([]string, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := cfg.GetMasterPublicAddress() + if err != nil { + return nil, err + } + kubePublicAddr, err := cfg.GetKubernetesPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr, err := cfg.GetAssetPublicAddress() + if err != nil { + return nil, err + } + + // 172.17.42.1 enables the router to call back out to the master + // TODO: Remove 172.17.42.1 once we can figure out how to validate the master's cert from inside a pod, or tell pods the real IP for the master + allHostnames := util.NewStringSet("localhost", "127.0.0.1", "172.17.42.1", masterAddr.Host, masterPublicAddr.Host, kubePublicAddr.Host, assetPublicAddr.Host) + certHostnames := util.StringSet{} + for hostname := range allHostnames { + if host, _, err := net.SplitHostPort(hostname); err == nil { + // add the hostname without the port + certHostnames.Insert(host) + } else { + // add the originally specified hostname + certHostnames.Insert(hostname) + } + } + + return certHostnames.List(), nil +} + +func (cfg Config) GetMasterCert() (certFile string, keyFile string, err error) { + ca, err := cfg.newCA() + if err != nil { + return "", "", err + } + + certHostnames, err := cfg.GetServerCertHostnames() + if err != nil { + return "", "", err + } + + serverCert, err := ca.MakeServerCert("master", certHostnames) + if err != nil { + return "", "", err + } + + return serverCert.CertFile, serverCert.KeyFile, nil +} + +func (cfg Config) GetAssetCert() (certFile string, keyFile string, err error) { + ca, err := cfg.newCA() + if err != nil { + return "", "", err + } + + certHostnames, err := cfg.GetServerCertHostnames() + if err != nil { + return "", "", err + } + + serverCert, err := ca.MakeServerCert("master", certHostnames) + if err != nil { + return "", "", err + } + + return serverCert.CertFile, serverCert.KeyFile, nil +} + +func (cfg Config) newClientConfigTemplate() (*clientcmdapi.Config, error) { + masterAddr, err := cfg.GetMasterAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := cfg.GetMasterPublicAddress() + if err != nil { + return nil, err + } + + return &clientcmdapi.Config{ + Clusters: map[string]clientcmdapi.Cluster{ + "master": {Server: masterAddr.String()}, + "public-master": {Server: masterPublicAddr.String()}, + }, + Contexts: map[string]clientcmdapi.Context{ + "master": {Cluster: "master"}, + "public-master": {Cluster: "public-master"}, + }, + CurrentContext: "master", + }, nil +} + +func (cfg Config) GetKubeClient() (*kclient.Client, *kclient.Config, error) { + var err error + var kubeClientConfig *kclient.Config + + // if we're starting an all in one, make credentials for a kube client. + if cfg.StartKube { + kubeClientConfig, err = cfg.MintSystemClientCert("kube-client") + if err != nil { + return nil, nil, err + } + + } else { + // we're pointing to an external kubernetes, so we'll use the passed kubeconfig + if cfg.ClientConfig == nil { + return nil, nil, errors.New("not starting an all-in-one, but no .kubeconfig specified") + } + kubeCfg, err := cfg.ClientConfig.ClientConfig() + if err != nil { + return nil, nil, err + } + + if len(kubeCfg.Version) == 0 { + kubeCfg.Version = klatest.Version + } + + kubeAddr, err := cfg.GetKubernetesAddress() + if err != nil { + return nil, nil, err + } + + kclient.SetKubernetesDefaults(kubeCfg) + kubeCfg.Host = kubeAddr.String() + + kubeClientConfig = kubeCfg + } + + kubeClient, err := kclient.New(kubeClientConfig) + if err != nil { + return nil, nil, err + } + + return kubeClient, kubeClientConfig, nil +} + +func (cfg Config) GetOpenshiftClient() (*osclient.Client, *kclient.Config, error) { + clientConfig, err := cfg.MintSystemClientCert("openshift-client") + if err != nil { + return nil, nil, err + } + + client, err := osclient.New(clientConfig) + if err != nil { + return nil, nil, err + } + + return client, clientConfig, nil +} + +func (cfg Config) GetOpenshiftDeployerClientConfig() (*kclient.Config, error) { + clientConfig, err := cfg.MintSystemClientCert("system:openshift-deployer", "system:deployers") + if err != nil { + return nil, err + } + + return clientConfig, nil +} + +// known certs: +// openshiftClientUser := &user.DefaultInfo{Name: "system:openshift-client"} +// openshiftDeployerUser := &user.DefaultInfo{Name: "system:openshift-deployer", Groups: []string{"system:deployers"}} +// adminUser := &user.DefaultInfo{Name: "system:admin", Groups: []string{"system:cluster-admins"}} +// kubeClientUser := &user.DefaultInfo{Name: "system:kube-client"} +// // One for each node in cfg.GetNodeList() +func (cfg Config) MintSystemClientCert(username string, groups ...string) (*kclient.Config, error) { + ca, err := cfg.newCA() + if err != nil { + return nil, err + } + clientConfigTemplate, err := cfg.newClientConfigTemplate() + if err != nil { + return nil, err + } + + qualifiedUsername := "system:" + username + user := &user.DefaultInfo{Name: qualifiedUsername, Groups: groups} + config, err := ca.MakeClientConfig(username, user, *clientConfigTemplate) + if err != nil { + return nil, err + } + + return &config, nil +} + +func (cfg Config) MintNodeCerts() error { + for _, node := range cfg.GetNodeList() { + username := "node-" + node + if _, err := cfg.MintSystemClientCert(username, "system:nodes"); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 8494366fa94e..cf9fbabee7ce 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -1,60 +1,22 @@ package server import ( - "crypto/x509" - "errors" "fmt" "net" "net/http" _ "net/http/pprof" + "net/url" "os" - "os/exec" "strconv" - "strings" - "time" - - "code.google.com/p/go-uuid/uuid" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" - "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" - kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" - clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" - kmaster "github.com/GoogleCloudPlatform/kubernetes/pkg/master" - "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" - kutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util" - "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" - etcdclient "github.com/coreos/go-etcd/etcd" "github.com/golang/glog" - "github.com/spf13/cobra" - - "github.com/openshift/origin/pkg/api/latest" - "github.com/openshift/origin/pkg/auth/authenticator" - "github.com/openshift/origin/pkg/auth/authenticator/request/bearertoken" - "github.com/openshift/origin/pkg/auth/authenticator/request/paramtoken" - "github.com/openshift/origin/pkg/auth/authenticator/request/unionrequest" - "github.com/openshift/origin/pkg/auth/authenticator/request/x509request" - "github.com/openshift/origin/pkg/authorization/authorizer" - policycache "github.com/openshift/origin/pkg/authorization/cache" - authorizationetcd "github.com/openshift/origin/pkg/authorization/registry/etcd" - "github.com/openshift/origin/pkg/authorization/rulevalidation" - projectauth "github.com/openshift/origin/pkg/project/auth" - - "github.com/openshift/origin/pkg/auth/group" - "github.com/openshift/origin/pkg/cmd/flagtypes" - "github.com/openshift/origin/pkg/cmd/server/crypto" + "github.com/openshift/origin/pkg/cmd/server/etcd" "github.com/openshift/origin/pkg/cmd/server/kubernetes" "github.com/openshift/origin/pkg/cmd/server/origin" - "github.com/openshift/origin/pkg/cmd/util" - cmdutil "github.com/openshift/origin/pkg/cmd/util" - "github.com/openshift/origin/pkg/cmd/util/docker" - "github.com/openshift/origin/pkg/cmd/util/variable" - pkgutil "github.com/openshift/origin/pkg/util" // Admission control plugins from upstream Kubernetes _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" @@ -64,39 +26,6 @@ import ( _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcequota" ) -const longCommandDesc = ` -Start an OpenShift server - -This command helps you launch an OpenShift server. The default mode is all-in-one, which allows -you to run all of the components of an OpenShift system on a server with Docker. Running - - $ openshift start - -will start OpenShift listening on all interfaces, launch an etcd server to store persistent -data, and launch the Kubernetes system components. The server will run in the foreground until -you terminate the process. - -Note: starting OpenShift without passing the --master address will attempt to find the IP -address that will be visible inside running Docker containers. This is not always successful, -so if you have problems tell OpenShift what public address it will be via --master=. - -You may also pass an optional argument to the start command to start OpenShift in one of the -following roles: - - $ openshift start master --nodes= - - Launches the server and control plane for OpenShift. You may pass a list of the node - hostnames you want to use, or create nodes via the REST API or 'openshift kube'. - - $ openshift start node --master= - - Launches a new node and attempts to connect to the master on the provided IP. - -You may also pass --etcd=
to connect to an external etcd server instead of running an -integrated instance, or --kubernetes= and --kubeconfig= to connect to an existing -Kubernetes cluster. -` - const ( unauthenticatedUsername = "system:anonymous" @@ -104,170 +33,102 @@ const ( unauthenticatedGroup = "system:unauthenticated" ) -// config is a struct that the command stores flag values into. -type config struct { - Docker *docker.Helper - - MasterAddr flagtypes.Addr - BindAddr flagtypes.Addr - EtcdAddr flagtypes.Addr - KubernetesAddr flagtypes.Addr - PortalNet flagtypes.IPNet - // addresses for external clients - MasterPublicAddr flagtypes.Addr - KubernetesPublicAddr flagtypes.Addr - - ImageFormat string - LatestReleaseImages bool - - ImageTemplate variable.ImageTemplate - - Hostname string - VolumeDir string - - EtcdDir string +func (cfg Config) startMaster() error { + // Allow privileged containers + // TODO: make this configurable and not the default https://github.com/openshift/origin/issues/662 + capabilities.Initialize(capabilities.Capabilities{ + AllowPrivileged: true, + }) - CertDir string + cfg.MintNodeCerts() + cfg.MintSystemClientCert("admin", "system:cluster-admins") + cfg.MintSystemClientCert("openshift-deployer", "system:deployers") + cfg.MintSystemClientCert("openshift-client") + cfg.MintSystemClientCert("kube-client") - StorageVersion string - - NodeList flagtypes.StringList - - // ClientConfig is used when connecting to Kubernetes from the master, or - // when connecting to the master from a detached node. If the server is an - // all-in-one, this value is not used. - ClientConfig clientcmd.ClientConfig - - CORSAllowedOrigins flagtypes.StringList -} - -// NewCommandStartServer provides a CLI handler for 'start' command -func NewCommandStartServer(name string) *cobra.Command { - hostname, err := defaultHostname() + openshiftConfig, err := cfg.BuildOriginMasterConfig() if err != nil { - hostname = "localhost" - glog.Warningf("Unable to lookup hostname, using %q: %v", hostname, err) + return err } + // must start policy caching immediately + openshiftConfig.RunPolicyCache() - cfg := &config{ - Docker: docker.NewHelper(), - - MasterAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - BindAddr: flagtypes.Addr{Value: "0.0.0.0:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - EtcdAddr: flagtypes.Addr{Value: "0.0.0.0:4001", DefaultScheme: "http", DefaultPort: 4001}.Default(), - KubernetesAddr: flagtypes.Addr{DefaultScheme: "https", DefaultPort: 8443}.Default(), - PortalNet: flagtypes.DefaultIPNet("172.30.17.0/24"), - MasterPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - KubernetesPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - - ImageTemplate: variable.NewDefaultImageTemplate(), - - Hostname: hostname, - NodeList: flagtypes.StringList{"127.0.0.1"}, + authConfig, err := cfg.BuildAuthConfig() + if err != nil { + return err } - cmd := &cobra.Command{ - Use: fmt.Sprintf("%s [master|node]", name), - Short: "Launch OpenShift", - Long: longCommandDesc, - Run: func(c *cobra.Command, args []string) { - if err := start(cfg, args); err != nil { - glog.Fatal(err) - } - }, - } + glog.Infof(" Nodes: %v", cfg.GetNodeList()) - flag := cmd.Flags() + if cfg.StartKube { + kubeConfig, err := cfg.BuildKubernetesMasterConfig(openshiftConfig.RequestContextMapper, openshiftConfig.KubeClient()) + if err != nil { + return err + } + kubeConfig.EnsurePortalFlags() - flag.Var(&cfg.BindAddr, "listen", "The address to listen for connections on (host, host:port, or URL).") - flag.Var(&cfg.MasterAddr, "master", "The master address for use by OpenShift components (host, host:port, or URL). Scheme and port default to the --listen scheme and port.") - flag.Var(&cfg.MasterPublicAddr, "public-master", "The master address for use by public clients, if different (host, host:port, or URL). Defaults to same as --master.") - flag.Var(&cfg.EtcdAddr, "etcd", "The address of the etcd server (host, host:port, or URL). If specified, no built-in etcd will be started.") - flag.Var(&cfg.KubernetesAddr, "kubernetes", "The address of the Kubernetes server (host, host:port, or URL). If specified, no Kubernetes components will be started.") - flag.Var(&cfg.KubernetesPublicAddr, "public-kubernetes", "The Kubernetes server address for use by public clients, if different. (host, host:port, or URL). Defaults to same as --kubernetes.") - flag.Var(&cfg.PortalNet, "portal-net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + openshiftConfig.Run([]origin.APIInstaller{kubeConfig}, []origin.APIInstaller{authConfig}) - flag.StringVar(&cfg.ImageTemplate.Format, "images", cfg.ImageTemplate.Format, "When fetching images used by the cluster for important components, use this format on both master and nodes. The latest release will be used by default.") - flag.BoolVar(&cfg.ImageTemplate.Latest, "latest-images", cfg.ImageTemplate.Latest, "If true, attempt to use the latest images for the cluster instead of the latest release.") + kubeConfig.RunScheduler() + kubeConfig.RunReplicationController() + kubeConfig.RunEndpointController() + kubeConfig.RunMinionController() + kubeConfig.RunResourceQuotaManager() - flag.StringVar(&cfg.VolumeDir, "volume-dir", "openshift.local.volumes", "The volume storage directory.") - flag.StringVar(&cfg.EtcdDir, "etcd-dir", "openshift.local.etcd", "The etcd data directory.") - flag.StringVar(&cfg.CertDir, "cert-dir", "openshift.local.certificates", "The certificate data directory.") + } else { + kubeAddr, err := cfg.GetKubernetesAddress() + if err != nil { + return err + } + proxy := &kubernetes.ProxyConfig{ + KubernetesAddr: kubeAddr, + ClientConfig: &openshiftConfig.KubeClientConfig, + } - flag.StringVar(&cfg.Hostname, "hostname", cfg.Hostname, "The hostname to identify this node with the master.") - flag.Var(&cfg.NodeList, "nodes", "The hostnames of each node. This currently must be specified up front. Comma delimited list") - flag.Var(&cfg.CORSAllowedOrigins, "cors-allowed-origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. CORS is enabled for localhost, 127.0.0.1, and the asset server by default.") + openshiftConfig.Run([]origin.APIInstaller{proxy}, []origin.APIInstaller{authConfig}) + } - cfg.ClientConfig = cmdutil.DefaultClientConfig(flag) + // TODO: recording should occur in individual components + record.StartRecording(openshiftConfig.KubeClient().Events(""), kapi.EventSource{Component: "master"}) - cfg.Docker.InstallFlags(flag) + openshiftConfig.RunAssetServer() + openshiftConfig.RunBuildController() + openshiftConfig.RunBuildImageChangeTriggerController() + openshiftConfig.RunDeploymentController() + openshiftConfig.RunDeploymentConfigController() + openshiftConfig.RunDeploymentConfigChangeController() + openshiftConfig.RunDeploymentImageChangeTriggerController() - return cmd + return nil } // run launches the appropriate startup modes or returns an error. -func start(cfg *config, args []string) error { - if len(args) > 1 { - return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") +func start(cfg Config, args []string) error { + if cfg.WriteConfigOnly { + return nil } - var startEtcd, startNode, startMaster, startKube bool - if len(args) == 1 { - switch args[0] { - case "master": - startMaster = true - startKube = !cfg.KubernetesAddr.Provided - startEtcd = !cfg.EtcdAddr.Provided - if err := defaultMasterAddress(cfg); err != nil { - return err - } - glog.Infof("Starting an OpenShift master, reachable at %s (etcd: %s)", cfg.MasterAddr.String(), cfg.EtcdAddr.String()) - if cfg.MasterPublicAddr.Provided { - glog.Infof("OpenShift master public address is %s", cfg.MasterPublicAddr.String()) - } - - case "node": - startNode = true - - // TODO client config currently doesn't let you override the defaults - // so it is defaulting to https://localhost:8443 for MasterAddr if - // it isn't set by --master or --kubeconfig - if !cfg.MasterAddr.Provided { - config, err := cfg.ClientConfig.ClientConfig() - if err != nil { - glog.Fatalf("Unable to read client configuration: %v", err) - } - if len(config.Host) > 0 { - cfg.MasterAddr.Set(config.Host) - } - } - if !cfg.KubernetesAddr.Provided { - cfg.KubernetesAddr = cfg.MasterAddr - } - glog.Infof("Starting an OpenShift node, connecting to %s", cfg.MasterAddr.String()) - - default: - return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") + switch { + case cfg.ExplicitStartMaster: + glog.Infof("Starting an OpenShift master, reachable at %s (etcd: %s)", cfg.MasterAddr.String(), cfg.EtcdAddr.String()) + if cfg.MasterPublicAddr.Provided { + glog.Infof("OpenShift master public address is %s", cfg.MasterPublicAddr.String()) } - } else { - startMaster = true - startKube = !cfg.KubernetesAddr.Provided - startEtcd = !cfg.EtcdAddr.Provided - startNode = true - if err := defaultMasterAddress(cfg); err != nil { - return err - } + case cfg.ExplicitStartNode: + glog.Infof("Starting an OpenShift node, connecting to %s", cfg.MasterAddr.String()) + default: glog.Infof("Starting an OpenShift all-in-one, reachable at %s (etcd: %s)", cfg.MasterAddr.String(), cfg.EtcdAddr.String()) if cfg.MasterPublicAddr.Provided { glog.Infof("OpenShift master public address is %s", cfg.MasterPublicAddr.String()) } } - if startKube { - cfg.KubernetesAddr = cfg.MasterAddr - cfg.KubernetesPublicAddr = cfg.MasterPublicAddr + if cfg.StartEtcd { + if err := cfg.RunEtcd(); err != nil { + return err + } } if env("OPENSHIFT_PROFILE", "") == "web" { @@ -277,391 +138,30 @@ func start(cfg *config, args []string) error { }() } - // define a function for resolving components to names - imageResolverFn := cfg.ImageTemplate.ExpandOrDie - - // the node can reuse an existing client - var existingKubeClient *kclient.Client - - if startMaster { - if len(cfg.NodeList) == 1 && cfg.NodeList[0] == "127.0.0.1" { - cfg.NodeList[0] = cfg.Hostname - } - for i, s := range cfg.NodeList { - s = strings.ToLower(s) - cfg.NodeList[i] = s - glog.Infof(" Node: %s", s) - } - - glog.Infof("Using images from %q", imageResolverFn("")) - - if startEtcd { - etcdConfig := &etcd.Config{ - BindAddr: cfg.BindAddr.Host, - PeerBindAddr: cfg.BindAddr.Host, - MasterAddr: cfg.EtcdAddr.URL.Host, - EtcdDir: cfg.EtcdDir, - } - etcdConfig.Run() - } - - // Connect and setup etcd interfaces - etcdClient, err := getEtcdClient(cfg) - if err != nil { + if cfg.StartMaster { + if err := cfg.startMaster(); err != nil { return err } - etcdHelper, err := origin.NewEtcdHelper(cfg.StorageVersion, etcdClient) - if err != nil { - return fmt.Errorf("Error setting up server storage: %v", err) - } - ketcdHelper, err := kmaster.NewEtcdHelper(etcdClient, klatest.Version) - if err != nil { - return fmt.Errorf("Error setting up Kubernetes server storage: %v", err) - } - - requestContextMapper := kapi.NewRequestContextMapper() - masterAuthorizationNamespace := "master" - - // determine whether public API addresses were specified - masterPublicAddr := cfg.MasterAddr - if cfg.MasterPublicAddr.Provided { - masterPublicAddr = cfg.MasterPublicAddr - } - k8sPublicAddr := masterPublicAddr // assume same place as master - if cfg.KubernetesPublicAddr.Provided { // specifically set, use that - k8sPublicAddr = cfg.KubernetesPublicAddr - } else if cfg.KubernetesAddr.Provided { // separate Kube, assume it is public - k8sPublicAddr = cfg.KubernetesAddr - } - - // Derive the asset bind address by incrementing the master bind address port by 1 - assetBindAddr := net.JoinHostPort(cfg.BindAddr.Host, strconv.Itoa(cfg.BindAddr.Port+1)) - // Derive the asset public address by incrementing the master public address port by 1 - assetPublicAddr := *masterPublicAddr.URL - assetPublicAddr.Host = net.JoinHostPort(masterPublicAddr.Host, strconv.Itoa(masterPublicAddr.Port+1)) - - // Build the list of valid redirect_uri prefixes for a login using the openshift-web-console client to redirect to - // TODO: allow configuring this - // TODO: remove hard-coding of development UI server - assetPublicAddresses := []string{assetPublicAddr.String(), "http://localhost:9000", "https://localhost:9000"} - - // always include the all-in-one server's web console as an allowed CORS origin - // always include localhost as an allowed CORS origin - // always include master public address as an allowed CORS origin - for _, origin := range []string{assetPublicAddr.Host, masterPublicAddr.URL.Host, "localhost", "127.0.0.1"} { - // TODO: check if origin is already allowed - cfg.CORSAllowedOrigins = append(cfg.CORSAllowedOrigins, origin) - } - - osmaster := &origin.MasterConfig{ - TLS: cfg.BindAddr.URL.Scheme == "https", - MasterBindAddr: cfg.BindAddr.URL.Host, - MasterAddr: cfg.MasterAddr.URL.String(), - MasterPublicAddr: masterPublicAddr.URL.String(), - AssetBindAddr: assetBindAddr, - AssetPublicAddr: assetPublicAddr.String(), - KubernetesAddr: cfg.KubernetesAddr.URL.String(), - KubernetesPublicAddr: k8sPublicAddr.URL.String(), - - LogoutURI: env("OPENSHIFT_LOGOUT_URI", ""), - - CORSAllowedOrigins: cfg.CORSAllowedOrigins, - - EtcdHelper: etcdHelper, - - AdmissionControl: admit.NewAlwaysAdmit(), - MasterAuthorizationNamespace: masterAuthorizationNamespace, - RequestContextMapper: requestContextMapper, - - ImageFor: imageResolverFn, - } - - if startKube { - // We're running against our own kubernetes server - osmaster.KubeClientConfig = kclient.Config{ - Host: cfg.MasterAddr.URL.String(), - Version: klatest.Version, - } - } else { - // We're running against another kubernetes server - osmaster.KubeClientConfig = *clientConfigFromKubeConfig(cfg) - } - - // Build token auth for user's OAuth tokens - authenticators := []authenticator.Request{} - tokenAuthenticator, err := origin.GetEtcdTokenAuthenticator(etcdHelper) - if err != nil { - glog.Fatalf("Error creating TokenAuthenticator: %v", err) - } - authenticators = append(authenticators, bearertoken.New(tokenAuthenticator, true)) - // Allow token as access_token param for WebSockets - // TODO: make the param name configurable - // TODO: limit this authenticator to watch methods, if possible - authenticators = append(authenticators, paramtoken.New("access_token", tokenAuthenticator, true)) - - var roots *x509.CertPool - if osmaster.TLS { - // Bootstrap CA - // TODO: store this (or parts of this) in etcd? - var err error - ca, err := crypto.InitCA(cfg.CertDir, fmt.Sprintf("%s@%d", cfg.MasterAddr.Host, time.Now().Unix())) - if err != nil { - return fmt.Errorf("Unable to configure certificate authority: %v", err) - } - - // Bootstrap server certs - // 172.17.42.1 enables the router to call back out to the master - // TODO: Remove 172.17.42.1 once we can figure out how to validate the master's cert from inside a pod, or tell pods the real IP for the master - allHostnames := []string{"localhost", "127.0.0.1", "172.17.42.1", cfg.MasterAddr.Host, masterPublicAddr.URL.Host, k8sPublicAddr.URL.Host, assetPublicAddr.Host} - certHostnames := []string{} - for _, hostname := range allHostnames { - if host, _, err := net.SplitHostPort(hostname); err == nil { - // add the hostname without the port - certHostnames = append(certHostnames, host) - } else { - // add the originally specified hostname - certHostnames = append(certHostnames, hostname) - } - } - serverCert, err := ca.MakeServerCert("master", pkgutil.UniqueStrings(certHostnames)) - if err != nil { - return err - } - osmaster.MasterCertFile = serverCert.CertFile - osmaster.MasterKeyFile = serverCert.KeyFile - osmaster.AssetCertFile = serverCert.CertFile - osmaster.AssetKeyFile = serverCert.KeyFile - - // Bootstrap clients - osClientConfigTemplate := clientcmdapi.Config{ - Clusters: map[string]clientcmdapi.Cluster{ - "master": {Server: cfg.MasterAddr.URL.String()}, - "public-master": {Server: masterPublicAddr.URL.String()}, - }, - Contexts: map[string]clientcmdapi.Context{ - "master": {Cluster: "master"}, - "public-master": {Cluster: "public-master"}, - }, - CurrentContext: "master", - } - - // OpenShift client - openshiftClientUser := &user.DefaultInfo{Name: "system:openshift-client"} - if osmaster.OSClientConfig, err = ca.MakeClientConfig("openshift-client", openshiftClientUser, osClientConfigTemplate); err != nil { - return err - } - // OpenShift deployer client - openshiftDeployerUser := &user.DefaultInfo{Name: "system:openshift-deployer", Groups: []string{"system:deployers"}} - if osmaster.DeployerOSClientConfig, err = ca.MakeClientConfig("openshift-deployer", openshiftDeployerUser, osClientConfigTemplate); err != nil { - return err - } - // Admin config (creates files on disk for osc) - adminUser := &user.DefaultInfo{Name: "system:admin", Groups: []string{"system:cluster-admins"}} - if _, err = ca.MakeClientConfig("admin", adminUser, osClientConfigTemplate); err != nil { - return err - } - - // One client config per node - for _, node := range cfg.NodeList { - nodeIdentityName := fmt.Sprintf("node-%s", node) - nodeUserName := fmt.Sprintf("system:%s", nodeIdentityName) - nodeUser := &user.DefaultInfo{Name: nodeUserName, Groups: []string{"system:nodes"}} - if _, err = ca.MakeClientConfig(nodeIdentityName, nodeUser, osClientConfigTemplate); err != nil { - return err - } - } - - // If we're running our own Kubernetes, build client credentials - if startKube { - kubeClientUser := &user.DefaultInfo{Name: "system:kube-client"} - if osmaster.KubeClientConfig, err = ca.MakeClientConfig("kube-client", kubeClientUser, osClientConfigTemplate); err != nil { - return err - } - } - - // Save cert roots - roots = x509.NewCertPool() - for _, root := range ca.Config.Roots { - roots.AddCert(root) - } - osmaster.ClientCAs = roots - - // build cert authenticator - // TODO: add cert users to etcd? - opts := x509request.DefaultVerifyOptions() - opts.Roots = roots - certauth := x509request.New(opts, x509request.SubjectToUserConversion) - authenticators = append(authenticators, certauth) - } else { - // No security, use the same client config for all OpenShift clients - osClientConfig := kclient.Config{Host: cfg.MasterAddr.URL.String(), Version: latest.Version} - osmaster.OSClientConfig = osClientConfig - osmaster.DeployerOSClientConfig = osClientConfig - } - - // TODO: make anonymous auth optional? - osmaster.Authenticator = &unionrequest.Authenticator{ - FailOnError: true, - Handlers: []authenticator.Request{ - group.NewGroupAdder(unionrequest.NewUnionAuthentication(authenticators...), []string{authenticatedGroup}), - authenticator.RequestFunc(func(req *http.Request) (user.Info, bool, error) { - return &user.DefaultInfo{Name: unauthenticatedUsername, Groups: []string{unauthenticatedGroup}}, true, nil - }), - }, - } - - osmaster.BuildClients() - - authorizationEtcd := authorizationetcd.New(etcdHelper) - osmaster.PolicyCache = policycache.NewPolicyCache(authorizationEtcd, authorizationEtcd) - osmaster.Authorizer = newAuthorizer(osmaster.PolicyCache, masterAuthorizationNamespace) - osmaster.AuthorizationAttributeBuilder = newAuthorizationAttributeBuilder(requestContextMapper) - // the policy cache must start before you attempt to start any other components - osmaster.RunPolicyCache() - - osmaster.ProjectAuthorizationCache = projectauth.NewAuthorizationCache( - projectauth.NewReviewer(osmaster.PolicyClient()), - osmaster.KubeClient().Namespaces(), - osmaster.PolicyClient(), - osmaster.PolicyClient(), - osmaster.MasterAuthorizationNamespace) - - // Default to a session authenticator (for browsers), and a basicauth authenticator (for clients responding to WWW-Authenticate challenges) - defaultAuthRequestHandlers := strings.Join([]string{ - string(origin.AuthRequestHandlerSession), - string(origin.AuthRequestHandlerBasicAuth), - }, ",") - auth := &origin.AuthConfig{ - MasterAddr: cfg.MasterAddr.URL.String(), - MasterPublicAddr: masterPublicAddr.URL.String(), - AssetPublicAddresses: assetPublicAddresses, - MasterRoots: roots, - EtcdHelper: etcdHelper, - - // Max token ages - AuthorizeTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_AUTHORIZE_TOKEN_MAX_AGE_SECONDS", 300, 1), - AccessTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_ACCESS_TOKEN_MAX_AGE_SECONDS", 3600, 1), - // Handlers - AuthRequestHandlers: origin.ParseAuthRequestHandlerTypes(env("OPENSHIFT_OAUTH_REQUEST_HANDLERS", defaultAuthRequestHandlers)), - AuthHandler: origin.AuthHandlerType(env("OPENSHIFT_OAUTH_HANDLER", string(origin.AuthHandlerLogin))), - GrantHandler: origin.GrantHandlerType(env("OPENSHIFT_OAUTH_GRANT_HANDLER", string(origin.GrantHandlerAuto))), - // RequestHeader config - RequestHeaders: strings.Split(env("OPENSHIFT_OAUTH_REQUEST_HEADERS", "X-Remote-User"), ","), - // Session config (default to unknowable secret) - SessionSecrets: []string{env("OPENSHIFT_OAUTH_SESSION_SECRET", uuid.NewUUID().String())}, - SessionMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_SESSION_MAX_AGE_SECONDS", 300, 1), - SessionName: env("OPENSHIFT_OAUTH_SESSION_NAME", "ssn"), - // Password config - PasswordAuth: origin.PasswordAuthType(env("OPENSHIFT_OAUTH_PASSWORD_AUTH", string(origin.PasswordAuthAnyPassword))), - BasicAuthURL: env("OPENSHIFT_OAUTH_BASIC_AUTH_URL", ""), - HTPasswdFile: env("OPENSHIFT_OAUTH_HTPASSWD_FILE", ""), - // Token config - TokenStore: origin.TokenStoreType(env("OPENSHIFT_OAUTH_TOKEN_STORE", string(origin.TokenStoreOAuth))), - TokenFilePath: env("OPENSHIFT_OAUTH_TOKEN_FILE_PATH", ""), - // Google config - GoogleClientID: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_ID", ""), - GoogleClientSecret: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_SECRET", ""), - // GitHub config - GithubClientID: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_ID", ""), - GithubClientSecret: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_SECRET", ""), - } - - // Allow privileged containers - // TODO: make this configurable and not the default https://github.com/openshift/origin/issues/662 - capabilities.Initialize(capabilities.Capabilities{ - AllowPrivileged: true, - }) - - if startKube { - portalNet := net.IPNet(cfg.PortalNet) - masterIP := net.ParseIP(cfg.MasterAddr.Host) - if masterIP == nil { - addrs, err := net.LookupIP(cfg.MasterAddr.Host) - if err != nil { - glog.Fatalf("Unable to find an IP for %q - specify an IP directly? %v", cfg.MasterAddr.Host, err) - } - if len(addrs) == 0 { - glog.Fatalf("Unable to find an IP for %q - specify an IP directly?", cfg.MasterAddr.Host) - } - masterIP = addrs[0] - } - - kmaster := &kubernetes.MasterConfig{ - MasterIP: masterIP, - MasterPort: cfg.MasterAddr.Port, - NodeHosts: cfg.NodeList, - PortalNet: &portalNet, - RequestContextMapper: requestContextMapper, - EtcdHelper: ketcdHelper, - KubeClient: osmaster.KubeClient(), - Authorizer: apiserver.NewAlwaysAllowAuthorizer(), - AdmissionControl: admit.NewAlwaysAdmit(), - } - kmaster.EnsurePortalFlags() - - osmaster.Run([]origin.APIInstaller{kmaster}, []origin.APIInstaller{auth}) - - kmaster.RunScheduler() - kmaster.RunReplicationController() - kmaster.RunEndpointController() - kmaster.RunMinionController() - kmaster.RunResourceQuotaManager() - - } else { - proxy := &kubernetes.ProxyConfig{ - KubernetesAddr: cfg.KubernetesAddr.URL, - ClientConfig: &osmaster.KubeClientConfig, - } - osmaster.Run([]origin.APIInstaller{proxy}, []origin.APIInstaller{auth}) - } - - // TODO: recording should occur in individual components - record.StartRecording(osmaster.KubeClient().Events(""), kapi.EventSource{Component: "master"}) - - osmaster.RunAssetServer() - osmaster.RunBuildController() - osmaster.RunBuildImageChangeTriggerController() - osmaster.RunDeploymentController() - osmaster.RunDeploymentConfigController() - osmaster.RunDeploymentConfigChangeController() - osmaster.RunDeploymentImageChangeTriggerController() - osmaster.RunProjectAuthorizationCache() - - existingKubeClient = osmaster.KubeClient() } - if startNode { - if existingKubeClient == nil { - config := clientConfigFromKubeConfig(cfg) - cli, err := kclient.New(config) - if err != nil { - glog.Fatalf("Unable to create a client: %v", err) - } - existingKubeClient = cli + if cfg.StartNode { + kubeClient, _, err := cfg.GetKubeClient() + if err != nil { + return err } - if !startMaster { + if !cfg.StartMaster { // TODO: recording should occur in individual components - record.StartRecording(existingKubeClient.Events(""), kapi.EventSource{Component: "node"}) + record.StartRecording(kubeClient.Events(""), kapi.EventSource{Component: "node"}) } - nodeConfig := &kubernetes.NodeConfig{ - BindHost: cfg.BindAddr.Host, - NodeHost: cfg.Hostname, - MasterHost: cfg.MasterAddr.URL.String(), - - VolumeDir: cfg.VolumeDir, - - NetworkContainerImage: imageResolverFn("pod"), - - AllowDisabledDocker: startNode && startMaster, - - Client: existingKubeClient, + nodeConfig, err := cfg.BuildKubernetesNodeConfig() + if err != nil { + return err } nodeConfig.EnsureVolumeDir() nodeConfig.EnsureDocker(cfg.Docker) - nodeConfig.RunProxy() nodeConfig.RunKubelet() } @@ -671,111 +171,6 @@ func start(cfg *config, args []string) error { return nil } -func newAuthorizer(policyCache *policycache.PolicyCache, masterAuthorizationNamespace string) authorizer.Authorizer { - authorizer := authorizer.NewAuthorizer(masterAuthorizationNamespace, rulevalidation.NewDefaultRuleResolver(policyCache, policyCache)) - return authorizer -} - -func newAuthorizationAttributeBuilder(requestContextMapper kapi.RequestContextMapper) authorizer.AuthorizationAttributeBuilder { - authorizationAttributeBuilder := authorizer.NewAuthorizationAttributeBuilder(requestContextMapper, &apiserver.APIRequestInfoResolver{kutil.NewStringSet("api", "osapi"), latest.RESTMapper}) - return authorizationAttributeBuilder -} - -// getEtcdClient creates an etcd client based on the provided config and waits -// until etcd server is reachable. It errors out and exits if the server cannot -// be reached for a certain amount of time. -func getEtcdClient(cfg *config) (*etcdclient.Client, error) { - etcdServers := []string{cfg.EtcdAddr.URL.String()} - etcdClient := etcdclient.NewClient(etcdServers) - - for i := 0; ; i++ { - _, err := etcdClient.Get("/", false, false) - if err == nil || tools.IsEtcdNotFound(err) { - break - } - if i > 100 { - return nil, fmt.Errorf("Could not reach etcd: %v", err) - } - time.Sleep(50 * time.Millisecond) - } - - return etcdClient, nil -} - -// defaultHostname returns the default hostname for this system. -func defaultHostname() (string, error) { - // Note: We use exec here instead of os.Hostname() because we - // want the FQDN, and this is the easiest way to get it. - fqdn, err := exec.Command("hostname", "-f").Output() - if err != nil { - return "", fmt.Errorf("Couldn't determine hostname: %v", err) - } - return strings.TrimSpace(string(fqdn)), nil -} - -// defaultMasterAddress checks for an unset master address and then attempts to use the first -// public IPv4 non-loopback address registered on this host. It will also update the -// EtcdAddr after if it was not provided. -// TODO: make me IPv6 safe -func defaultMasterAddress(cfg *config) error { - if !cfg.MasterAddr.Provided { - // If the user specifies a bind address, and the master is not provided, use the bind port by default - port := cfg.MasterAddr.Port - if cfg.BindAddr.Provided { - port = cfg.BindAddr.Port - } - - // If the user specifies a bind address, and the master is not provided, use the bind scheme by default - scheme := cfg.MasterAddr.URL.Scheme - if cfg.BindAddr.Provided { - scheme = cfg.BindAddr.URL.Scheme - } - - // use the default ip address for the system - addr, err := util.DefaultLocalIP4() - if err != nil { - return fmt.Errorf("Unable to find the public address of this master: %v", err) - } - - masterAddr := scheme + "://" + net.JoinHostPort(addr.String(), strconv.Itoa(port)) - if err := cfg.MasterAddr.Set(masterAddr); err != nil { - return fmt.Errorf("Unable to set public address of this master: %v", err) - } - - // Prefer to use the MasterAddr for etcd because some clients still connect to it. - if !cfg.EtcdAddr.Provided { - etcdAddr := net.JoinHostPort(addr.String(), strconv.Itoa(cfg.EtcdAddr.DefaultPort)) - if err := cfg.EtcdAddr.Set(etcdAddr); err != nil { - return fmt.Errorf("Unable to set public address of etcd: %v", err) - } - } - } else if !cfg.EtcdAddr.Provided { - // Etcd should be reachable on the same address that the master is (for simplicity) - etcdAddr := net.JoinHostPort(cfg.MasterAddr.Host, strconv.Itoa(cfg.EtcdAddr.DefaultPort)) - if err := cfg.EtcdAddr.Set(etcdAddr); err != nil { - return fmt.Errorf("Unable to set public address of etcd: %v", err) - } - } - return nil -} - -// clientConfigFromKubeConfig reads the client configuration settings for connecting to -// a Kubernetes master. -func clientConfigFromKubeConfig(cfg *config) *kclient.Config { - config, err := cfg.ClientConfig.ClientConfig() - if err != nil { - glog.Fatalf("Unable to read client configuration: %v", err) - } - if len(config.Version) == 0 { - config.Version = klatest.Version - } - kclient.SetKubernetesDefaults(config) - if cfg.KubernetesAddr.Provided { - config.Host = cfg.KubernetesAddr.URL.String() - } - return config -} - func envInt(key string, defaultValue int32, minValue int32) int32 { value, err := strconv.ParseInt(env(key, fmt.Sprintf("%d", defaultValue)), 10, 32) if err != nil || int32(value) < minValue { @@ -794,33 +189,39 @@ func env(key string, defaultValue string) string { return val } -func expandImage(component, defaultValue string, latest bool) string { - template := defaultValue - if s, ok := imageComponentEnvExpander(component); ok { - template = s +func (cfg Config) RunEtcd() error { + etcdAddr, err := cfg.GetEtcdAddress() + if err != nil { + return err } - value, err := variable.ExpandStrict(template, func(key string) (string, bool) { - switch key { - case "component": - return component, true - case "version": - if latest { - return "latest", true - } - } - return "", false - }, variable.Versions) + + etcdConfig := &etcd.Config{ + BindAddr: cfg.GetEtcdBindAddress(), + PeerBindAddr: cfg.GetEtcdPeerBindAddress(), + MasterAddr: etcdAddr.Host, + EtcdDir: cfg.EtcdDir, + } + + etcdConfig.Run() + + return nil +} + +func getHost(theURL url.URL) string { + host, _, err := net.SplitHostPort(theURL.Host) if err != nil { - glog.Fatalf("Unable to find an image for %q due to an error processing the format: %v", err) + return theURL.Host } - return value + + return host } -func imageComponentEnvExpander(key string) (string, bool) { - s := strings.Replace(strings.ToUpper(key), "-", "_", -1) - val := os.Getenv(fmt.Sprintf("OPENSHIFT_%s_IMAGE", s)) - if len(val) == 0 { - return "", false +func getPort(theURL url.URL) int { + _, port, err := net.SplitHostPort(theURL.Host) + if err != nil { + return 0 } - return val, true + + intport, _ := strconv.Atoi(port) + return intport } diff --git a/pkg/cmd/server/start_test.go b/pkg/cmd/server/start_test.go deleted file mode 100644 index fb176eacdcdc..000000000000 --- a/pkg/cmd/server/start_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package server - -import ( - "os" - "testing" - - "github.com/openshift/origin/pkg/cmd/util/variable" - "github.com/openshift/origin/pkg/version" -) - -func TestExpandDefaultImage(t *testing.T) { - variable.OverrideVersion = version.Get() - variable.OverrideVersion.GitVersion = "v1.0" - - os.Setenv("OPENSHIFT_COMPONENT_IMAGE", "test") - - tests := []struct { - component string - template string - latest bool - output string - }{ - {"*", "openshift/origin-${component}", true, "openshift/origin-*"}, - {"version", "openshift/origin-${component}", true, "openshift/origin-version"}, - {"version", "openshift/origin-${component}:${version}", true, "openshift/origin-version:latest"}, - {"version", "openshift/origin-${component}:${version}", false, "openshift/origin-version:v1.0"}, - {"component", "openshift/origin-${component}:${version}", true, "test"}, - } - for _, test := range tests { - if s := expandImage(test.component, test.template, test.latest); s != test.output { - t.Errorf("unexpected image expansion for %#v: %s", test, s) - } - } -}