diff --git a/hack/test-cmd.sh b/hack/test-cmd.sh index 4c541dc47a9a..e88bf8c32b73 100755 --- a/hack/test-cmd.sh +++ b/hack/test-cmd.sh @@ -75,7 +75,7 @@ OPENSHIFT_ON_PANIC=crash openshift start --master="${API_SCHEME}://${API_HOST}:$ OS_PID=$! if [[ "${API_SCHEME}" == "https" ]]; then - export CURL_CA_BUNDLE="${CERT_DIR}/admin/root.crt" + export CURL_CA_BUNDLE="${CERT_DIR}/ca/cert.crt" export CURL_CERT="${CERT_DIR}/admin/cert.crt" export CURL_KEY="${CERT_DIR}/admin/key.key" fi @@ -174,14 +174,18 @@ echo "templates: ok" [ "$(osc get --help 2>&1 | grep 'Display one or many resources')" ] [ "$(openshift cli get --help 2>&1 | grep 'Display one or many resources')" ] [ "$(openshift kubectl get --help 2>&1 | grep 'Display one or many resources')" ] -[ "$(openshift start --help 2>&1 | grep 'Start an OpenShift server')" ] +[ "$(openshift start --help 2>&1 | grep 'Start an OpenShift all-in-one server')" ] +[ "$(openshift start master --help 2>&1 | grep 'Start an OpenShift master')" ] +[ "$(openshift start node --help 2>&1 | grep 'Start an OpenShift node')" ] [ "$(osc get --help 2>&1 | grep 'osc')" ] # help for given command through help command must be consistent [ "$(osc help get 2>&1 | grep 'Display one or many resources')" ] [ "$(openshift cli help get 2>&1 | grep 'Display one or many resources')" ] [ "$(openshift kubectl help get 2>&1 | grep 'Display one or many resources')" ] -[ "$(openshift help start 2>&1 | grep 'Start an OpenShift server')" ] +[ "$(openshift help start 2>&1 | grep 'Start an OpenShift all-in-one server')" ] +[ "$(openshift help start master 2>&1 | grep 'Start an OpenShift master')" ] +[ "$(openshift help start node 2>&1 | grep 'Start an OpenShift node')" ] [ "$(openshift cli help update 2>&1 | grep 'openshift')" ] # runnable commands with required flags must error consistently diff --git a/hack/test-end-to-end.sh b/hack/test-end-to-end.sh index 7b394998d342..cd3811f5f5f2 100755 --- a/hack/test-end-to-end.sh +++ b/hack/test-end-to-end.sh @@ -50,9 +50,12 @@ LOG_DIR="${LOG_DIR:-${BASETMPDIR}/logs}" ARTIFACT_DIR="${ARTIFACT_DIR:-${BASETMPDIR}/artifacts}" mkdir -p $LOG_DIR mkdir -p $ARTIFACT_DIR + +DEFAULT_SERVER_IP=`ifconfig | grep -Ev "(127.0.0.1|172.17.42.1)" | grep "inet " | head -n 1 | awk '{print $2}'` +API_HOST="${API_HOST:-${DEFAULT_SERVER_IP}}" API_PORT="${API_PORT:-8443}" API_SCHEME="${API_SCHEME:-https}" -API_HOST="${API_HOST:-localhost}" +MASTER_ADDR="${API_SCHEME}://${API_HOST}:${API_PORT}" PUBLIC_MASTER_HOST="${PUBLIC_MASTER_HOST:-${API_HOST}}" KUBELET_SCHEME="${KUBELET_SCHEME:-http}" KUBELET_PORT="${KUBELET_PORT:-10250}" @@ -168,25 +171,32 @@ echo "[INFO] Using images: ${USE_IMAGES}" # Start All-in-one server and wait for health # Specify the scheme and port for the listen address, but let the IP auto-discover. Set --public-master to localhost, for a stable link to the console. +echo "[INFO] Create certificates for the OpenShift server" +# find the same IP that openshift start will bind to. This allows access from pods that have to talk back to master +ALL_IP_ADDRESSES=`ifconfig | grep "inet " | awk '{print $2}'` +SERVER_HOSTNAME_LIST="${PUBLIC_MASTER_HOST},localhost" +while read -r IP_ADDRESS +do + SERVER_HOSTNAME_LIST="${SERVER_HOSTNAME_LIST},${IP_ADDRESS}" +done <<< "${ALL_IP_ADDRESSES}" + +openshift admin create-all-certs --overwrite=false --cert-dir="${CERT_DIR}" --hostnames="${SERVER_HOSTNAME_LIST}" --nodes="127.0.0.1" --master="${MASTER_ADDR}" --public-master="${API_SCHEME}://${PUBLIC_MASTER_HOST}" + + echo "[INFO] Starting OpenShift server" sudo env "PATH=${PATH}" OPENSHIFT_ON_PANIC=crash openshift start \ - --listen="${API_SCHEME}://0.0.0.0:${API_PORT}" --public-master="${API_SCHEME}://${PUBLIC_MASTER_HOST}" \ + --listen="${API_SCHEME}://0.0.0.0:${API_PORT}" --master="${MASTER_ADDR}" --public-master="${API_SCHEME}://${PUBLIC_MASTER_HOST}" \ --hostname="127.0.0.1" --volume-dir="${VOLUME_DIR}" \ --etcd-dir="${ETCD_DATA_DIR}" --cert-dir="${CERT_DIR}" --loglevel=4 \ - --images="${USE_IMAGES}" \ + --images="${USE_IMAGES}" --create-certs=false\ &> "${LOG_DIR}/openshift.log" & OS_PID=$! if [[ "${API_SCHEME}" == "https" ]]; then - export CURL_CA_BUNDLE="${CERT_DIR}/master/root.crt" + export CURL_CA_BUNDLE="${CERT_DIR}/ca/cert.crt" export CURL_CERT="${CERT_DIR}/admin/cert.crt" export CURL_KEY="${CERT_DIR}/admin/key.key" - # Generate the certs first - wait_for_file "${CERT_DIR}/openshift-client/key.key" 0.5 80 - wait_for_file "${CERT_DIR}/admin/key.key" 0.5 80 - wait_for_file "${CURL_CA_BUNDLE}" 0.5 80 - # Read client cert data in to send to containerized components sudo chmod -R a+rX "${CERT_DIR}/openshift-client/" @@ -204,6 +214,9 @@ wait_for_url "${API_SCHEME}://${API_HOST}:${API_PORT}/api/v1beta1/minions/127.0. # Set KUBERNETES_MASTER for osc export KUBERNETES_MASTER="${API_SCHEME}://${API_HOST}:${API_PORT}" +# add e2e-user as a viewer for the default namespace so we can see infrastructure pieces appear +openshift ex policy add-user view anypassword:e2e-user --namespace=default + # create test project so that this shows up in the console openshift ex new-project test --description="This is an example project to demonstrate OpenShift v3" --admin="anypassword:e2e-user" diff --git a/hack/test-integration.sh b/hack/test-integration.sh index 82b5114b9b7a..4727c3692854 100755 --- a/hack/test-integration.sh +++ b/hack/test-integration.sh @@ -44,12 +44,23 @@ start_etcd trap cleanup EXIT SIGINT function exectest() { + echo "running $1..." + out=$("${testexec}" -test.run="^$1$" "${@:2}" 2>&1) + + tput cuu 1 # Move up one line + tput el # Clear "running" line + res=$? if [[ ${res} -eq 0 ]]; then - echo ok $1 + tput setaf 2 # green + echo "ok $1" + tput sgr0 # reset exit 0 else + tput setaf 1 # red + echo "failed $1" + tput sgr0 # reset echo "${out}" exit 1 fi diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh index 690f78e3b1bc..075f6b0e11b9 100755 --- a/hack/verify-gofmt.sh +++ b/hack/verify-gofmt.sh @@ -25,6 +25,7 @@ find_files() { \( \ -wholename './output' \ -o -wholename './_output' \ + -o -wholename './.git' \ -o -wholename './release' \ -o -wholename './pkg/assets/bindata.go' \ -o -wholename './target' \ diff --git a/pkg/authorization/authorizer/subjects_test.go b/pkg/authorization/authorizer/subjects_test.go index 792c2e1bf052..175956b53294 100644 --- a/pkg/authorization/authorizer/subjects_test.go +++ b/pkg/authorization/authorizer/subjects_test.go @@ -33,7 +33,7 @@ func TestSubjects(t *testing.T) { Resource: "pods", }, expectedUsers: util.NewStringSet("Anna", "ClusterAdmin", "Ellen", "Valerie", "system:kube-client", "system:openshift-client", "system:openshift-deployer"), - expectedGroups: util.NewStringSet("RootUsers", "system:cluster-admins"), + expectedGroups: util.NewStringSet("RootUsers", "system:cluster-admins", "system:nodes"), } test.policies = newDefaultGlobalPolicies() test.policies = append(test.policies, newAdzePolicies()...) diff --git a/pkg/cmd/openshift/openshift.go b/pkg/cmd/openshift/openshift.go index bee62d15306b..3d6eef200c01 100644 --- a/pkg/cmd/openshift/openshift.go +++ b/pkg/cmd/openshift/openshift.go @@ -18,7 +18,8 @@ import ( "github.com/openshift/origin/pkg/cmd/infra/builder" "github.com/openshift/origin/pkg/cmd/infra/deployer" "github.com/openshift/origin/pkg/cmd/infra/router" - "github.com/openshift/origin/pkg/cmd/server" + "github.com/openshift/origin/pkg/cmd/server/certs" + "github.com/openshift/origin/pkg/cmd/server/start" "github.com/openshift/origin/pkg/cmd/templates" "github.com/openshift/origin/pkg/cmd/util/clientcmd" "github.com/openshift/origin/pkg/version" @@ -78,8 +79,9 @@ func NewCommandOpenShift() *cobra.Command { root.SetUsageTemplate(templates.MainUsageTemplate()) root.SetHelpTemplate(templates.MainHelpTemplate()) - openshiftStartCommand, _ := server.NewCommandStartServer("start") - root.AddCommand(openshiftStartCommand) + startAllInOne, _ := start.NewCommandStartAllInOne() + root.AddCommand(startAllInOne) + root.AddCommand(certs.NewCommandAdmin()) root.AddCommand(cli.NewCommandCLI("cli", "openshift cli")) root.AddCommand(cli.NewCmdKubectl("kube")) root.AddCommand(newExperimentalCommand("openshift", "ex")) diff --git a/pkg/cmd/server/api/helpers.go b/pkg/cmd/server/api/helpers.go new file mode 100644 index 000000000000..7df3cb592c4e --- /dev/null +++ b/pkg/cmd/server/api/helpers.go @@ -0,0 +1,125 @@ +package api + +import ( + "crypto/x509" + "fmt" + "io/ioutil" + + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + + "github.com/openshift/origin/pkg/client" + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +func GetKubeClient(kubeConfigFile string) (*kclient.Client, *kclient.Config, error) { + loadingRules := &clientcmd.ClientConfigLoadingRules{CommandLinePath: kubeConfigFile} + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + + kubeConfig, err := loader.ClientConfig() + if err != nil { + return nil, nil, err + } + kubeClient, err := kclient.New(kubeConfig) + if err != nil { + return nil, nil, err + } + + return kubeClient, kubeConfig, nil +} + +func GetOpenShiftClient(kubeConfigFile string) (*client.Client, *kclient.Config, error) { + loadingRules := &clientcmd.ClientConfigLoadingRules{CommandLinePath: kubeConfigFile} + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) + + kubeConfig, err := loader.ClientConfig() + if err != nil { + return nil, nil, err + } + openshiftClient, err := client.New(kubeConfig) + if err != nil { + return nil, nil, err + } + + return openshiftClient, kubeConfig, nil +} + +func UseTLS(servingInfo ServingInfo) bool { + return len(servingInfo.ServerCert.CertFile) > 0 +} + +// GetAPIClientCertCAPool returns the cert pool used to validate client certificates to the API server +func GetAPIClientCertCAPool(options MasterConfig) (*x509.CertPool, error) { + certs, err := getAPIClientCertCAs(options) + if err != nil { + return nil, err + } + roots := x509.NewCertPool() + for _, root := range certs { + roots.AddCert(root) + } + return roots, nil +} + +// GetClientCertCAPool returns a cert pool containing all client CAs that could be presented (union of API and OAuth) +func GetClientCertCAPool(options MasterConfig) (*x509.CertPool, error) { + roots := x509.NewCertPool() + + // Add CAs for OAuth + certs, err := getOAuthClientCertCAs(options) + if err != nil { + return nil, err + } + for _, root := range certs { + roots.AddCert(root) + } + + // Add CAs for API + certs, err = getAPIClientCertCAs(options) + if err != nil { + return nil, err + } + for _, root := range certs { + roots.AddCert(root) + } + + return roots, nil +} + +// GetAPIServerCertCAPool returns the cert pool containing the roots for the API server cert +func GetAPIServerCertCAPool(options MasterConfig) (*x509.CertPool, error) { + caRoots, err := crypto.GetTLSCARoots(options.ServingInfo.ClientCA) + if err != nil { + return nil, err + } + roots := x509.NewCertPool() + for _, root := range caRoots.Roots { + roots.AddCert(root) + } + return roots, nil +} + +func getOAuthClientCertCAs(options MasterConfig) ([]*x509.Certificate, error) { + caFile := options.OAuthConfig.ProxyCA + if len(caFile) == 0 { + return nil, nil + } + caPEMBlock, err := ioutil.ReadFile(caFile) + if err != nil { + return nil, err + } + certs, err := crypto.CertsFromPEM(caPEMBlock) + if err != nil { + return nil, fmt.Errorf("Error reading %s: %s", caFile, err) + } + return certs, nil +} + +func getAPIClientCertCAs(options MasterConfig) ([]*x509.Certificate, error) { + apiClientCertCAs, err := crypto.GetTLSCARoots(options.ServingInfo.ClientCA) + if err != nil { + return nil, err + } + + return apiClientCertCAs.Roots, nil +} diff --git a/pkg/cmd/server/api/latest/latest.go b/pkg/cmd/server/api/latest/latest.go new file mode 100644 index 000000000000..491dfd194920 --- /dev/null +++ b/pkg/cmd/server/api/latest/latest.go @@ -0,0 +1,24 @@ +package latest + +import ( + "github.com/openshift/origin/pkg/cmd/server/api/v1" +) + +// Version is the string that represents the current external default version. +const Version = "v1" + +// OldestVersion is the string that represents the oldest server version supported, +// for client code that wants to hardcode the lowest common denominator. +const OldestVersion = "v1" + +// Versions is the list of versions that are recognized in code. The order provided +// may be assumed to be least feature rich to most feature rich, and clients may +// choose to prefer the latter items in the list over the former items when presented +// with a set of versions to choose. +var Versions = []string{"v1"} + +// Codec is the default codec for serializing output that should use +// the latest supported version. Use this Codec when writing to +// disk, a data store that is not dynamically versioned, or in tests. +// This codec can decode any object that Kubernetes is aware of. +var Codec = v1.Codec diff --git a/pkg/cmd/server/api/register.go b/pkg/cmd/server/api/register.go new file mode 100644 index 000000000000..e2eda68c5ae1 --- /dev/null +++ b/pkg/cmd/server/api/register.go @@ -0,0 +1,17 @@ +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" +) + +var Scheme = runtime.NewScheme() + +func init() { + Scheme.AddKnownTypes("", + &MasterConfig{}, + &NodeConfig{}, + ) +} + +func (*MasterConfig) IsAnAPIObject() {} +func (*NodeConfig) IsAnAPIObject() {} diff --git a/pkg/cmd/server/api/types.go b/pkg/cmd/server/api/types.go new file mode 100644 index 000000000000..4bf621a45edd --- /dev/null +++ b/pkg/cmd/server/api/types.go @@ -0,0 +1,161 @@ +package api + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" +) + +// NodeConfig is the fully specified config starting an OpenShift node +type NodeConfig struct { + api.TypeMeta + + // NodeName is the value used to identify this particular node in the cluster. If possible, this should be your fully qualified hostname. + // If you're describing a set of static nodes to the master, this value must match one of the values in the list + NodeName string + + // ServingInfo describes how to start serving + ServingInfo ServingInfo + + // MasterKubeConfig is a filename for the .kubeconfig file that describes how to connect this node to the master + MasterKubeConfig string + + // domain suffix + DNSDomain string + // ip + DNSIP string + + // VolumeDir is the directory that volumes will be stored under + VolumeDirectory string + + // NetworkContainerImage is the image used as the Kubelet network namespace and volume container. + NetworkContainerImage string + + // AllowDisabledDocker if true, the Kubelet will ignore errors from Docker. This means that a node can start on a machine that doesn't have docker started. + AllowDisabledDocker bool + + // RecordEvents indicates whether or not to record events from the master + RecordEvents bool +} + +type MasterConfig struct { + api.TypeMeta + + // ServingInfo describes how to start serving + ServingInfo ServingInfo + + // CORSAllowedOrigins + CORSAllowedOrigins []string + + // EtcdClientInfo contains information about how to connect to etcd + EtcdClientInfo RemoteConnectionInfo + + // KubernetesMasterConfig, if present start the kubernetes master in this process + KubernetesMasterConfig *KubernetesMasterConfig + // EtcdConfig, if present start etcd in this process + EtcdConfig *EtcdConfig + // OAuthConfig, if present start the /oauth endpoint in this process + OAuthConfig *OAuthConfig + // AssetConfig, if present start the asset serverin this process + AssetConfig *AssetConfig + // DNSConfig, if present start the DNS server in this process + DNSConfig *DNSConfig + + // MasterClients holds all the client connection information for controllers and other system components + MasterClients MasterClients + + ImageConfig ImageConfig + + // MasterAuthorizationNamespace is the global namespace for Policy + MasterAuthorizationNamespace string + // OpenShiftSharedResourcesNamespace is the namespace where shared OpenShift resources live (like shared templates) + OpenShiftSharedResourcesNamespace string +} + +type ImageConfig struct { + Format string + Latest bool +} + +type RemoteConnectionInfo struct { + // URL is the URL for etcd + URL string + // CA is the CA for confirming that the server at the etcdURL is the actual server + CA string + // EtcdClientCertInfo is the TLS client cert information for securing communication to etcd + // this is anonymous so that we can inline it for serialization + ClientCert CertInfo +} + +type ServingInfo struct { + // BindAddress is the ip:port to serve on + BindAddress string + // ServerCert is the TLS cert info for serving secure traffic + ServerCert CertInfo + // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates + ClientCA string +} + +type MasterClients struct { + // DeployerKubeConfig is a .kubeconfig filename for depoyment pods to use + DeployerKubeConfig string + // OpenShiftLoopbackKubeConfig is a .kubeconfig filename for system components to loopback to this master + OpenShiftLoopbackKubeConfig string + // KubernetesKubeConfig is a .kubeconfig filename for system components to communicate to kubernetes for building the proxy + KubernetesKubeConfig string +} + +type DNSConfig struct { + // BindAddress is the ip:port to serve DNS on + BindAddress string +} + +type AssetConfig struct { + ServingInfo ServingInfo + + // PublicURL is where you can find the asset server (TODO do we really need this?) + PublicURL string + + // LogoutURI is an optional, absolute URI to redirect web browsers to after logging out of the web console. + // If not specified, the built-in logout page is shown. + LogoutURI string + + // MasterPublicURL is how the web console can access the OpenShift api server + MasterPublicURL string + + // TODO: we probably don't need this since we have a proxy + // KubernetesPublicURL is how the web console can access the Kubernetes api server + KubernetesPublicURL string +} + +type OAuthConfig struct { + // ProxyCA is the certificate bundle for confirming the identity of front proxy forwards to the oauth server + ProxyCA string + + // MasterURL is used for building valid client redirect URLs for external access + MasterURL string + + // MasterPublicURL is used for building valid client redirect URLs for external access + MasterPublicURL string + + // AssetPublicURL is used for building valid client redirect URLs for external access + AssetPublicURL string + + // all the handlers here +} + +type EtcdConfig struct { + ServingInfo ServingInfo + + PeerAddress string + MasterAddress string + StorageDir string +} + +type KubernetesMasterConfig struct { + ServicesSubnet string + StaticNodeNames []string +} + +type CertInfo struct { + CertFile string + KeyFile string +} diff --git a/pkg/cmd/server/api/v1/conversions.go b/pkg/cmd/server/api/v1/conversions.go new file mode 100644 index 000000000000..663ebe5fa453 --- /dev/null +++ b/pkg/cmd/server/api/v1/conversions.go @@ -0,0 +1,43 @@ +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/conversion" + newer "github.com/openshift/origin/pkg/cmd/server/api" +) + +func init() { + err := newer.Scheme.AddConversionFuncs( + func(in *ServingInfo, out *newer.ServingInfo, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.ClientCA = in.ClientCA + out.ServerCert.CertFile = in.CertFile + out.ServerCert.KeyFile = in.KeyFile + return nil + }, + func(in *newer.ServingInfo, out *ServingInfo, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.ClientCA = in.ClientCA + out.CertFile = in.ServerCert.CertFile + out.KeyFile = in.ServerCert.KeyFile + return nil + }, + func(in *RemoteConnectionInfo, out *newer.RemoteConnectionInfo, s conversion.Scope) error { + out.URL = in.URL + out.CA = in.CA + out.ClientCert.CertFile = in.CertFile + out.ClientCert.KeyFile = in.KeyFile + return nil + }, + func(in *newer.RemoteConnectionInfo, out *RemoteConnectionInfo, s conversion.Scope) error { + out.URL = in.URL + out.CA = in.CA + out.CertFile = in.ClientCert.CertFile + out.KeyFile = in.ClientCert.KeyFile + return nil + }, + ) + if err != nil { + // If one of the conversion functions is malformed, detect it immediately. + panic(err) + } +} diff --git a/pkg/cmd/server/api/v1/register.go b/pkg/cmd/server/api/v1/register.go new file mode 100644 index 000000000000..1c296ff3c95b --- /dev/null +++ b/pkg/cmd/server/api/v1/register.go @@ -0,0 +1,18 @@ +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" + "github.com/openshift/origin/pkg/cmd/server/api" +) + +var Codec = runtime.CodecFor(api.Scheme, "v1") + +func init() { + api.Scheme.AddKnownTypes("v1", + &MasterConfig{}, + &NodeConfig{}, + ) +} + +func (*MasterConfig) IsAnAPIObject() {} +func (*NodeConfig) IsAnAPIObject() {} diff --git a/pkg/cmd/server/api/v1/types.go b/pkg/cmd/server/api/v1/types.go new file mode 100644 index 000000000000..5db5367759ed --- /dev/null +++ b/pkg/cmd/server/api/v1/types.go @@ -0,0 +1,162 @@ +package v1 + +import ( + "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta3" +) + +// NodeConfig is the fully specified config starting an OpenShift node +type NodeConfig struct { + v1beta3.TypeMeta `json:",inline"` + + // NodeName is the value used to identify this particular node in the cluster. If possible, this should be your fully qualified hostname. + // If you're describing a set of static nodes to the master, this value must match one of the values in the list + NodeName string `json:"nodeName"` + + // ServingInfo describes how to start serving + ServingInfo ServingInfo `json:"servingInfo"` + + // MasterKubeConfig is a filename for the .kubeconfig file that describes how to connect this node to the master + MasterKubeConfig string `json:"masterKubeConfig"` + + // domain suffix + DNSDomain string `json:"dnsDomain"` + // ip + DNSIP string `json:"dnsIP"` + + // VolumeDir is the directory that volumes will be stored under + VolumeDirectory string `json:"volumeDirectory"` + + // NetworkContainerImage is the image used as the Kubelet network namespace and volume container. + NetworkContainerImage string `json:"networkContainerImage"` + + // AllowDisabledDocker if true, the Kubelet will ignore errors from Docker. This means that a node can start on a machine that doesn't have docker started. + AllowDisabledDocker bool `json:"allowDisabledDocker"` + + // RecordEvents indicates whether or not to record events from the master + RecordEvents bool `json:"recordEvents"` +} + +type MasterConfig struct { + v1beta3.TypeMeta `json:",inline"` + + // ServingInfo describes how to start serving + ServingInfo ServingInfo `json:"servingInfo"` + + // CORSAllowedOrigins + CORSAllowedOrigins []string `json:"corsAllowedOrigins"` + + // EtcdClientInfo contains information about how to connect to etcd + EtcdClientInfo RemoteConnectionInfo `json:"etcdClientInfo"` + + // KubernetesMasterConfig, if present start the kubernetes master in this process + KubernetesMasterConfig *KubernetesMasterConfig `json:"kubernetesMasterConfig"` + // EtcdConfig, if present start etcd in this process + EtcdConfig *EtcdConfig `json:"etcdConfig"` + // OAuthConfig, if present start the /oauth endpoint in this process + OAuthConfig *OAuthConfig `json:"oauthConfig"` + // AssetConfig, if present start the asset serverin this process + AssetConfig *AssetConfig `json:"assetConfig"` + // DNSConfig, if present start the DNS server in this process + DNSConfig *DNSConfig `json:"dnsConfig"` + + // MasterClients holds all the client connection information for controllers and other system components + MasterClients MasterClients `json:"masterClients"` + + ImageConfig ImageConfig `json:"imageConfig"` + + // MasterAuthorizationNamespace is the global namespace for Policy + MasterAuthorizationNamespace string `json:"masterAuthorizationNamespace"` + // OpenShiftSharedResourcesNamespace is the namespace where shared OpenShift resources live (like shared templates) + OpenShiftSharedResourcesNamespace string `json:"openshiftSharedResourcesNamespace"` +} + +type ImageConfig struct { + Format string `json:"format"` + Latest bool `json:"latest"` +} + +type RemoteConnectionInfo struct { + // URL is the URL for etcd + URL string `json:"url"` + // CA is the CA for confirming that the server at the etcdURL is the actual server + CA string `json:"ca"` + // EtcdClientCertInfo is the TLS client cert information for securing communication to etcd + // this is anonymous so that we can inline it for serialization + CertInfo `json:",inline"` +} + +type ServingInfo struct { + // BindAddress is the ip:port to serve on + BindAddress string `json:"bindAddress"` + // ServerCert is the TLS cert info for serving secure traffic. + // this is anonymous so that we can inline it for serialization + CertInfo `json:",inline"` + // ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates + ClientCA string `json:"clientCA"` +} + +type MasterClients struct { + // DeployerKubeConfig is a .kubeconfig filename for depoyment pods to use + DeployerKubeConfig string `json:"deployerKubeConfig"` + // OpenShiftLoopbackKubeConfig is a .kubeconfig filename for system components to loopback to this master + OpenShiftLoopbackKubeConfig string `json:"openshiftLoopbackKubeConfig"` + // KubernetesKubeConfig is a .kubeconfig filename for system components to communicate to kubernetes for building the proxy + KubernetesKubeConfig string `json:"kubernetesKubeConfig"` +} + +type DNSConfig struct { + // BindAddress is the ip:port to serve DNS on + BindAddress string `json:"bindAddress"` +} + +type AssetConfig struct { + ServingInfo ServingInfo `json:"servingInfo"` + + // PublicURL is where you can find the asset server (TODO do we really need this?) + PublicURL string `json:"publicURL"` + + // LogoutURI is an optional, absolute URI to redirect web browsers to after logging out of the web console. + // If not specified, the built-in logout page is shown. + LogoutURI string `json:"logoutURI"` + + // MasterPublicURL is how the web console can access the OpenShift api server + MasterPublicURL string `json:"masterPublicURL"` + + // TODO: we probably don't need this since we have a proxy + // KubernetesPublicURL is how the web console can access the Kubernetes api server + KubernetesPublicURL string `json:"kubernetesPublicURL"` +} + +type OAuthConfig struct { + // ProxyCA is the certificate bundle for confirming the identity of front proxy forwards to the oauth server + ProxyCA string `json:"proxyCA"` + + // MasterURL is used for building valid client redirect URLs for external access + MasterURL string `json:"masterURL"` + + // MasterPublicURL is used for building valid client redirect URLs for external access + MasterPublicURL string `json:"masterPublicURL"` + + // AssetPublicURL is used for building valid client redirect URLs for external access + AssetPublicURL string `json:"assetPublicURL"` + + // all the handlers here +} + +type EtcdConfig struct { + ServingInfo ServingInfo `json:"servingInfo"` + + PeerAddress string `json:"peerAddress"` + MasterAddress string `json:"masterAddress"` + StorageDir string `json:"storageDirectory"` +} + +type KubernetesMasterConfig struct { + ServicesSubnet string `json:"servicesSubnet"` + StaticNodeNames []string `json:"staticNodeNames"` +} + +type CertInfo struct { + CertFile string `json:"certFile"` + KeyFile string `json:"keyFile"` +} diff --git a/pkg/cmd/server/bootstrappolicy/policy.go b/pkg/cmd/server/bootstrappolicy/policy.go index 17587efeb644..cac396e4361a 100644 --- a/pkg/cmd/server/bootstrappolicy/policy.go +++ b/pkg/cmd/server/bootstrappolicy/policy.go @@ -16,6 +16,7 @@ const ( AuthenticatedGroup = "system:authenticated" UnauthenticatedGroup = "system:unauthenticated" ClusterAdminGroup = "system:cluster-admins" + NodesGroup = "system:nodes" ) const ( @@ -226,7 +227,8 @@ func GetBootstrapMasterRoleBindings(masterNamespace string) []authorizationapi.R Name: InternalComponentRoleName, Namespace: masterNamespace, }, - Users: util.NewStringSet(InternalComponentUsername, InternalComponentKubeUsername), + Users: util.NewStringSet(InternalComponentUsername, InternalComponentKubeUsername), + Groups: util.NewStringSet(NodesGroup), }, { ObjectMeta: kapi.ObjectMeta{ diff --git a/pkg/cmd/server/certs/create_allcerts.go b/pkg/cmd/server/certs/create_allcerts.go new file mode 100644 index 000000000000..20d9120760e4 --- /dev/null +++ b/pkg/cmd/server/certs/create_allcerts.go @@ -0,0 +1,182 @@ +package certs + +import ( + "errors" + "fmt" + "path" + "path/filepath" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" +) + +type CreateAllCertsOptions struct { + CertDir string + SignerName string + + Hostnames util.StringList + NodeList util.StringList + + APIServerURL string + PublicAPIServerURL string + + Overwrite bool +} + +func NewCommandCreateAllCerts() *cobra.Command { + options := &CreateAllCertsOptions{} + + cmd := &cobra.Command{ + Use: "create-all-certs", + Short: "Create all certificates for OpenShift All-In-One", + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if err := options.CreateAllCerts(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + + flags.StringVar(&options.CertDir, "cert-dir", "openshift.local.certificates", "The certificate data directory.") + flags.StringVar(&options.SignerName, "signer-name", DefaultSignerName(), "The name to use for the generated signer.") + + flags.StringVar(&options.APIServerURL, "master", "https://localhost:8443", "The API server's URL.") + flags.StringVar(&options.PublicAPIServerURL, "public-master", "", "The API public facing server's URL (if applicable).") + flags.Var(&options.Hostnames, "hostnames", "Every hostname or IP you want server certs to be valid for. Comma delimited list") + flags.Var(&options.NodeList, "nodes", "The names of all static nodes you'd like to generate certificates for. Comma delimited list") + flags.BoolVar(&options.Overwrite, "overwrite", true, "Overwrite existing cert files if found. If false, any existing file will be left as-is.") + + return cmd +} + +func (o CreateAllCertsOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.Hostnames) == 0 { + return errors.New("at least one hostname must be provided") + } + if len(o.CertDir) == 0 { + return errors.New("cert-dir must be provided") + } + if len(o.SignerName) == 0 { + return errors.New("signer-name must be provided") + } + if len(o.APIServerURL) == 0 { + return errors.New("master must be provided") + } + + return nil +} + +func (o CreateAllCertsOptions) CreateAllCerts() error { + glog.V(2).Infof("Creating all certs with: %#v", o) + + signerCertOptions := CreateSignerCertOptions{ + CertFile: DefaultCertFilename(o.CertDir, DefaultCADir), + KeyFile: DefaultKeyFilename(o.CertDir, DefaultCADir), + SerialFile: DefaultSerialFilename(o.CertDir, DefaultCADir), + Name: o.SignerName, + Overwrite: o.Overwrite, + } + if _, err := signerCertOptions.CreateSignerCert(); err != nil { + return err + } + // once we've minted the signer, don't overwrite it + getSignerCertOptions := GetSignerCertOptions{ + CertFile: DefaultCertFilename(o.CertDir, DefaultCADir), + KeyFile: DefaultKeyFilename(o.CertDir, DefaultCADir), + SerialFile: DefaultSerialFilename(o.CertDir, DefaultCADir), + } + + for _, clientCertInfo := range DefaultClientCerts(o.CertDir) { + clientCertOptions := CreateClientCertOptions{ + GetSignerCertOptions: &getSignerCertOptions, + + CertFile: clientCertInfo.CertLocation.CertFile, + KeyFile: clientCertInfo.CertLocation.KeyFile, + + User: clientCertInfo.User, + Groups: util.StringList(clientCertInfo.Groups.List()), + Overwrite: o.Overwrite, + } + if _, err := clientCertOptions.CreateClientCert(); err != nil { + return err + } + + createKubeConfigOptions := CreateKubeConfigOptions{ + APIServerURL: o.APIServerURL, + PublicAPIServerURL: o.PublicAPIServerURL, + APIServerCAFile: getSignerCertOptions.CertFile, + ServerNick: "master", + + CertFile: clientCertInfo.CertLocation.CertFile, + KeyFile: clientCertInfo.CertLocation.KeyFile, + UserNick: clientCertInfo.SubDir, + + KubeConfigFile: path.Join(filepath.Dir(clientCertOptions.CertFile), ".kubeconfig"), + } + if _, err := createKubeConfigOptions.CreateKubeConfig(); err != nil { + return err + } + } + + for _, nodeName := range o.NodeList { + username := "node-" + nodeName + nodeCertOptions := CreateNodeClientCertOptions{ + GetSignerCertOptions: &getSignerCertOptions, + + CertFile: DefaultCertFilename(o.CertDir, username), + KeyFile: DefaultKeyFilename(o.CertDir, username), + + NodeName: nodeName, + Overwrite: o.Overwrite, + } + if _, err := nodeCertOptions.CreateNodeClientCert(); err != nil { + return err + } + + createKubeConfigOptions := CreateKubeConfigOptions{ + APIServerURL: o.APIServerURL, + PublicAPIServerURL: o.PublicAPIServerURL, + APIServerCAFile: getSignerCertOptions.CertFile, + ServerNick: "master", + + CertFile: nodeCertOptions.CertFile, + KeyFile: nodeCertOptions.KeyFile, + UserNick: username, + + KubeConfigFile: path.Join(filepath.Dir(nodeCertOptions.CertFile), ".kubeconfig"), + } + if _, err := createKubeConfigOptions.CreateKubeConfig(); err != nil { + return err + } + } + + for _, serverCertInfo := range DefaultServerCerts(o.CertDir) { + serverCertOptions := CreateServerCertOptions{ + GetSignerCertOptions: &getSignerCertOptions, + + CertFile: serverCertInfo.CertFile, + KeyFile: serverCertInfo.KeyFile, + + Hostnames: o.Hostnames, + Overwrite: o.Overwrite, + } + + if _, err := serverCertOptions.CreateServerCert(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/server/certs/create_clientcert.go b/pkg/cmd/server/certs/create_clientcert.go new file mode 100644 index 000000000000..c842ce2bffa6 --- /dev/null +++ b/pkg/cmd/server/certs/create_clientcert.go @@ -0,0 +1,93 @@ +package certs + +import ( + "errors" + "fmt" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +type CreateClientCertOptions struct { + GetSignerCertOptions *GetSignerCertOptions + + CertFile string + KeyFile string + + User string + Groups util.StringList + + Overwrite bool +} + +func NewCommandCreateClientCert() *cobra.Command { + options := &CreateClientCertOptions{GetSignerCertOptions: &GetSignerCertOptions{}} + + cmd := &cobra.Command{ + Use: "create-client-cert", + Short: "Create client certificate", + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if _, err := options.CreateClientCert(); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + }, + } + + flags := cmd.Flags() + BindGetSignerCertOptions(options.GetSignerCertOptions, flags, "") + + flags.StringVar(&options.CertFile, "cert", "openshift.local.certificates/user/cert.crt", "The certificate file.") + flags.StringVar(&options.KeyFile, "key", "openshift.local.certificates/user/key.key", "The key file.") + + flags.StringVar(&options.User, "user", "", "The scope qualified username.") + flags.Var(&options.Groups, "groups", "The list of groups this user belongs to. Comma delimited list") + flags.BoolVar(&options.Overwrite, "overwrite", true, "Overwrite existing cert files if found. If false, any existing file will be left as-is.") + + return cmd +} + +func (o CreateClientCertOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.CertFile) == 0 { + return errors.New("cert must be provided") + } + if len(o.KeyFile) == 0 { + return errors.New("key must be provided") + } + if len(o.User) == 0 { + return errors.New("user must be provided") + } + + return o.GetSignerCertOptions.Validate() +} + +func (o CreateClientCertOptions) CreateClientCert() (*crypto.TLSCertificateConfig, error) { + glog.V(2).Infof("Createing a client cert with: %#v and %#v", o, o.GetSignerCertOptions) + + signerCert, err := o.GetSignerCertOptions.GetSignerCert() + if err != nil { + return nil, err + } + + userInfo := &user.DefaultInfo{Name: o.User, Groups: o.Groups} + if o.Overwrite { + return signerCert.MakeClientCertificate(o.CertFile, o.KeyFile, userInfo) + } else { + return signerCert.EnsureClientCertificate(o.CertFile, o.KeyFile, userInfo) + } +} diff --git a/pkg/cmd/server/certs/create_comands.go b/pkg/cmd/server/certs/create_comands.go new file mode 100644 index 000000000000..d64f5b84270e --- /dev/null +++ b/pkg/cmd/server/certs/create_comands.go @@ -0,0 +1,24 @@ +package certs + +import ( + "github.com/spf13/cobra" +) + +func NewCommandAdmin() *cobra.Command { + cmd := &cobra.Command{ + Use: "admin", + Short: "Admin commands", + Run: func(c *cobra.Command, args []string) { + c.Help() + }, + } + + cmd.AddCommand(NewCommandCreateKubeConfig()) + cmd.AddCommand(NewCommandCreateAllCerts()) + cmd.AddCommand(NewCommandCreateClientCert()) + cmd.AddCommand(NewCommandCreateNodeClientCert()) + cmd.AddCommand(NewCommandCreateServerCert()) + cmd.AddCommand(NewCommandCreateSignerCert()) + + return cmd +} diff --git a/pkg/cmd/server/certs/create_kubeconfig.go b/pkg/cmd/server/certs/create_kubeconfig.go new file mode 100644 index 000000000000..67d5777f0f65 --- /dev/null +++ b/pkg/cmd/server/certs/create_kubeconfig.go @@ -0,0 +1,162 @@ +package certs + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" +) + +type CreateKubeConfigOptions struct { + APIServerURL string + PublicAPIServerURL string + APIServerCAFile string + ServerNick string + + CertFile string + KeyFile string + UserNick string + + KubeConfigFile string +} + +func NewCommandCreateKubeConfig() *cobra.Command { + options := &CreateKubeConfigOptions{} + + cmd := &cobra.Command{ + Use: "create-kubeconfig", + Short: "Create a basic .kubeconfig file from client certs", + Long: ` +Create's a .kubeconfig file at <--kubeconfig> that looks like this: + +clusters: +- cluster: + certificate-authority-data: + server: <--master> + name: <--cluster> +- cluster: + certificate-authority-data: + server: <--public-master> + name: public-<--cluster> +contexts: +- context: + cluster: <--cluster> + user: <--user> + name: <--cluster> +current-context: <--cluster> +kind: Config +users: +- name: <--user> + user: + client-certificate-data: + client-key-data: +`, + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if _, err := options.CreateKubeConfig(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + + flags.StringVar(&options.APIServerURL, "master", "https://localhost:8443", "The API server's URL.") + flags.StringVar(&options.PublicAPIServerURL, "public-master", "", "The API public facing server's URL (if applicable).") + flags.StringVar(&options.APIServerCAFile, "certificate-authority", "openshift.local.certificates/ca/cert.crt", "Path to the API server's CA file.") + flags.StringVar(&options.ServerNick, "cluster", "master", "Nick name for this server in .kubeconfig.") + flags.StringVar(&options.CertFile, "client-certificate", "openshift.local.certificates/admin/cert.crt", "The client cert file.") + flags.StringVar(&options.KeyFile, "client-key", "openshift.local.certificates/admin/key.key", "The client key file.") + flags.StringVar(&options.UserNick, "user", "user", "Nick name for this user in .kubeconfig.") + flags.StringVar(&options.KubeConfigFile, "kubeconfig", ".kubeconfig", "Path for the resulting .kubeconfig file.") + + return cmd +} + +func (o CreateKubeConfigOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.KubeConfigFile) == 0 { + return errors.New("kubeconfig must be provided") + } + if len(o.ServerNick) == 0 { + return errors.New("cluster must be provided") + } + if len(o.UserNick) == 0 { + return errors.New("user-nick must be provided") + } + + return nil +} + +func (o CreateKubeConfigOptions) CreateKubeConfig() (*clientcmdapi.Config, error) { + glog.V(2).Infof("creating a .kubeconfig with: %#v", o) + + caData, err := ioutil.ReadFile(o.APIServerCAFile) + if err != nil { + return nil, err + } + certData, err := ioutil.ReadFile(o.CertFile) + if err != nil { + return nil, err + } + keyData, err := ioutil.ReadFile(o.KeyFile) + if err != nil { + return nil, err + } + + credentials := make(map[string]clientcmdapi.AuthInfo) + credentials[o.UserNick] = clientcmdapi.AuthInfo{ + ClientCertificateData: certData, + ClientKeyData: keyData, + } + + clusters := make(map[string]clientcmdapi.Cluster) + clusters[o.ServerNick] = clientcmdapi.Cluster{ + Server: o.APIServerURL, + CertificateAuthorityData: caData, + } + + contexts := make(map[string]clientcmdapi.Context) + contexts[o.ServerNick] = clientcmdapi.Context{Cluster: o.ServerNick, AuthInfo: o.UserNick} + + createPublic := len(o.PublicAPIServerURL) > 0 + if createPublic { + publicNick := "public-" + o.ServerNick + clusters[publicNick] = clientcmdapi.Cluster{ + Server: o.PublicAPIServerURL, + CertificateAuthorityData: caData, + } + contexts[publicNick] = clientcmdapi.Context{Cluster: o.ServerNick, AuthInfo: o.UserNick} + } + + kubeConfig := &clientcmdapi.Config{ + Clusters: clusters, + AuthInfos: credentials, + Contexts: contexts, + CurrentContext: o.ServerNick, + } + + // Ensure the parent dir exists + if err := os.MkdirAll(filepath.Dir(o.KubeConfigFile), os.FileMode(0755)); err != nil { + return nil, err + } + if err := clientcmd.WriteToFile(*kubeConfig, o.KubeConfigFile); err != nil { + return nil, err + } + + return kubeConfig, nil +} diff --git a/pkg/cmd/server/certs/create_nodeclientcerts.go b/pkg/cmd/server/certs/create_nodeclientcerts.go new file mode 100644 index 000000000000..82e9bff68a43 --- /dev/null +++ b/pkg/cmd/server/certs/create_nodeclientcerts.go @@ -0,0 +1,91 @@ +package certs + +import ( + "errors" + "fmt" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +type CreateNodeClientCertOptions struct { + GetSignerCertOptions *GetSignerCertOptions + + CertFile string + KeyFile string + + NodeName string + + Overwrite bool +} + +func NewCommandCreateNodeClientCert() *cobra.Command { + options := &CreateNodeClientCertOptions{GetSignerCertOptions: &GetSignerCertOptions{}} + + cmd := &cobra.Command{ + Use: "create-node-cert", + Short: "Create node certificate", + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if _, err := options.CreateNodeClientCert(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + BindGetSignerCertOptions(options.GetSignerCertOptions, flags, "") + + flags.StringVar(&options.CertFile, "cert", "openshift.local.certificates/user/cert.crt", "The certificate file.") + flags.StringVar(&options.KeyFile, "key", "openshift.local.certificates/user/key.key", "The key file.") + + flags.StringVar(&options.NodeName, "node-name", "", "The name of the node.") + flags.BoolVar(&options.Overwrite, "overwrite", true, "Overwrite existing cert files if found. If false, any existing file will be left as-is.") + + return cmd +} + +func (o CreateNodeClientCertOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.CertFile) == 0 { + return errors.New("cert must be provided") + } + if len(o.KeyFile) == 0 { + return errors.New("key must be provided") + } + if len(o.NodeName) == 0 { + return errors.New("node-name must be provided") + } + + return o.GetSignerCertOptions.Validate() +} + +func (o CreateNodeClientCertOptions) CreateNodeClientCert() (*crypto.TLSCertificateConfig, error) { + glog.V(2).Infof("Createing a node client cert with: %#v and %#v", o, o.GetSignerCertOptions) + + username := "node-" + o.NodeName + + nodeCertOptions := CreateClientCertOptions{ + GetSignerCertOptions: o.GetSignerCertOptions, + + CertFile: o.CertFile, + KeyFile: o.KeyFile, + + User: "system:" + username, + Groups: util.StringList([]string{"system:nodes"}), + Overwrite: o.Overwrite, + } + + return nodeCertOptions.CreateClientCert() +} diff --git a/pkg/cmd/server/certs/create_servercert.go b/pkg/cmd/server/certs/create_servercert.go new file mode 100644 index 000000000000..3e439aab7e7f --- /dev/null +++ b/pkg/cmd/server/certs/create_servercert.go @@ -0,0 +1,86 @@ +package certs + +import ( + "errors" + "fmt" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +type CreateServerCertOptions struct { + GetSignerCertOptions *GetSignerCertOptions + + CertFile string + KeyFile string + + Hostnames util.StringList + Overwrite bool +} + +func NewCommandCreateServerCert() *cobra.Command { + options := &CreateServerCertOptions{GetSignerCertOptions: &GetSignerCertOptions{}} + + cmd := &cobra.Command{ + Use: "create-server-cert", + Short: "Create server certificate", + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if _, err := options.CreateServerCert(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + BindGetSignerCertOptions(options.GetSignerCertOptions, flags, "signer-") + + flags.StringVar(&options.CertFile, "cert", "openshift.local.certificates/user/cert.crt", "The certificate file.") + flags.StringVar(&options.KeyFile, "key", "openshift.local.certificates/user/key.key", "The key file.") + + flags.Var(&options.Hostnames, "hostnames", "Every hostname or IP you want server certs to be valid for. Comma delimited list") + flags.BoolVar(&options.Overwrite, "overwrite", true, "Overwrite existing cert files if found. If false, any existing file will be left as-is.") + + return cmd +} + +func (o CreateServerCertOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.Hostnames) == 0 { + return errors.New("at least one hostname must be provided") + } + if len(o.CertFile) == 0 { + return errors.New("cert must be provided") + } + if len(o.KeyFile) == 0 { + return errors.New("key must be provided") + } + + return o.GetSignerCertOptions.Validate() +} + +func (o CreateServerCertOptions) CreateServerCert() (*crypto.TLSCertificateConfig, error) { + glog.V(2).Infof("Createing a server cert with: %#v", o) + + signerCert, err := o.GetSignerCertOptions.GetSignerCert() + if err != nil { + return nil, err + } + + if o.Overwrite { + return signerCert.MakeServerCert(o.CertFile, o.KeyFile, util.NewStringSet([]string(o.Hostnames)...)) + } else { + return signerCert.EnsureServerCert(o.CertFile, o.KeyFile, util.NewStringSet([]string(o.Hostnames)...)) + } +} diff --git a/pkg/cmd/server/certs/create_signercert.go b/pkg/cmd/server/certs/create_signercert.go new file mode 100644 index 000000000000..13794c1985ee --- /dev/null +++ b/pkg/cmd/server/certs/create_signercert.go @@ -0,0 +1,83 @@ +package certs + +import ( + "errors" + "fmt" + + "github.com/golang/glog" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +type CreateSignerCertOptions struct { + CertFile string + KeyFile string + SerialFile string + Name string + + Overwrite bool +} + +func BindSignerCertOptions(options *CreateSignerCertOptions, flags *pflag.FlagSet, prefix string) { + flags.StringVar(&options.CertFile, prefix+"cert", "openshift.local.certificates/ca/cert.crt", "The certificate file.") + flags.StringVar(&options.KeyFile, prefix+"key", "openshift.local.certificates/ca/key.key", "The key file.") + flags.StringVar(&options.SerialFile, prefix+"serial", "openshift.local.certificates/ca/serial.txt", "The serial file that keeps track of how many certs have been signed.") + flags.StringVar(&options.Name, prefix+"name", DefaultSignerName(), "The name of the signer.") + flags.BoolVar(&options.Overwrite, prefix+"overwrite", options.Overwrite, "Overwrite existing cert files if found. If false, any existing file will be left as-is.") +} + +func NewCommandCreateSignerCert() *cobra.Command { + options := &CreateSignerCertOptions{Overwrite: true} + + cmd := &cobra.Command{ + Use: "create-signer-cert", + Short: "Create signer certificate", + Run: func(c *cobra.Command, args []string) { + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if _, err := options.CreateSignerCert(); err != nil { + glog.Fatal(err) + } + }, + } + + BindSignerCertOptions(options, cmd.Flags(), "") + + return cmd +} + +func (o CreateSignerCertOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + if len(o.CertFile) == 0 { + return errors.New("cert must be provided") + } + if len(o.KeyFile) == 0 { + return errors.New("key must be provided") + } + if len(o.SerialFile) == 0 { + return errors.New("serial must be provided") + } + if len(o.Name) == 0 { + return errors.New("name must be provided") + } + + return nil +} + +func (o CreateSignerCertOptions) CreateSignerCert() (*crypto.CA, error) { + glog.V(2).Infof("Createing a signer cert with: %#v", o) + + if o.Overwrite { + return crypto.MakeCA(o.CertFile, o.KeyFile, o.SerialFile, o.Name) + } else { + return crypto.EnsureCA(o.CertFile, o.KeyFile, o.SerialFile, o.Name) + } +} diff --git a/pkg/cmd/server/certs/default_certs.go b/pkg/cmd/server/certs/default_certs.go new file mode 100644 index 000000000000..1626303e957d --- /dev/null +++ b/pkg/cmd/server/certs/default_certs.go @@ -0,0 +1,131 @@ +package certs + +import ( + "fmt" + "path" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" +) + +const ( + DefaultCADir = "ca" +) + +type ClientCertInfo struct { + CertLocation configapi.CertInfo + SubDir string + User string + Groups util.StringSet +} + +func DefaultSignerName() string { + return fmt.Sprintf("%s@%d", "openshift-signer", time.Now().Unix()) +} + +func DefaultRootCAFile(certDir string) string { + return DefaultCertFilename(certDir, DefaultCADir) +} + +func DefaultClientCerts(certDir string) []ClientCertInfo { + return []ClientCertInfo{ + DefaultDeployerClientCertInfo(certDir), + DefaultOpenshiftLoopbackClientCertInfo(certDir), + DefaultKubeClientClientCertInfo(certDir), + DefaultClusterAdminClientCertInfo(certDir), + } +} + +func DefaultDeployerClientCertInfo(certDir string) ClientCertInfo { + return ClientCertInfo{ + CertLocation: configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "openshift-deployer"), + KeyFile: DefaultKeyFilename(certDir, "openshift-deployer"), + }, + SubDir: "openshift-deployer", + User: "system:openshift-deployer", + Groups: util.NewStringSet("system:deployers"), + } +} + +func DefaultOpenshiftLoopbackClientCertInfo(certDir string) ClientCertInfo { + return ClientCertInfo{ + CertLocation: configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "openshift-client"), + KeyFile: DefaultKeyFilename(certDir, "openshift-client"), + }, + SubDir: "openshift-client", + User: "system:openshift-client", + } +} + +func DefaultKubeClientClientCertInfo(certDir string) ClientCertInfo { + return ClientCertInfo{ + CertLocation: configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "kube-client"), + KeyFile: DefaultKeyFilename(certDir, "kube-client"), + }, + SubDir: "kube-client", + User: "system:kube-client", + } +} + +func DefaultClusterAdminClientCertInfo(certDir string) ClientCertInfo { + return ClientCertInfo{ + CertLocation: configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "admin"), + KeyFile: DefaultKeyFilename(certDir, "admin"), + }, + SubDir: "admin", + User: "system:admin", + Groups: util.NewStringSet("system:cluster-admins"), + } +} + +func DefaultServerCerts(certDir string) []configapi.CertInfo { + return []configapi.CertInfo{ + DefaultMasterServingCertInfo(certDir), + DefaultAssetServingCertInfo(certDir), + } +} + +func DefaultMasterServingCertInfo(certDir string) configapi.CertInfo { + return configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "master"), + KeyFile: DefaultKeyFilename(certDir, "master"), + } +} + +func DefaultNodeServingCertInfo(certDir, nodeName string) configapi.CertInfo { + return configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "node_serving-"+nodeName), + KeyFile: DefaultKeyFilename(certDir, "node_serving-"+nodeName), + } +} + +func DefaultAssetServingCertInfo(certDir string) configapi.CertInfo { + return configapi.CertInfo{ + CertFile: DefaultCertFilename(certDir, "master"), + KeyFile: DefaultKeyFilename(certDir, "master"), + } +} + +func DefaultCertDir(certDir, username string) string { + return path.Join(certDir, username) +} + +func DefaultCertFilename(certDir, username string) string { + return path.Join(DefaultCertDir(certDir, username), "cert.crt") +} + +func DefaultKeyFilename(certDir, username string) string { + return path.Join(DefaultCertDir(certDir, username), "key.key") +} +func DefaultSerialFilename(certDir, username string) string { + return path.Join(DefaultCertDir(certDir, username), "serial.txt") +} +func DefaultKubeConfigFilename(certDir, username string) string { + return path.Join(DefaultCertDir(certDir, username), ".kubeconfig") +} diff --git a/pkg/cmd/server/certs/signer_cert_args.go b/pkg/cmd/server/certs/signer_cert_args.go new file mode 100644 index 000000000000..8d342afc18c2 --- /dev/null +++ b/pkg/cmd/server/certs/signer_cert_args.go @@ -0,0 +1,39 @@ +package certs + +import ( + "errors" + + "github.com/spf13/pflag" + + "github.com/openshift/origin/pkg/cmd/server/crypto" +) + +type GetSignerCertOptions struct { + CertFile string + KeyFile string + SerialFile string +} + +func BindGetSignerCertOptions(options *GetSignerCertOptions, flags *pflag.FlagSet, prefix string) { + flags.StringVar(&options.CertFile, prefix+"signer-cert", "openshift.local.certificates/ca/cert.crt", "The certificate file.") + flags.StringVar(&options.KeyFile, prefix+"signer-key", "openshift.local.certificates/ca/key.key", "The key file.") + flags.StringVar(&options.SerialFile, prefix+"signer-serial", "openshift.local.certificates/ca/serial.txt", "The serial file that keeps track of how many certs have been signed.") +} + +func (o GetSignerCertOptions) Validate() error { + if len(o.CertFile) == 0 { + return errors.New("signer-cert must be provided") + } + if len(o.KeyFile) == 0 { + return errors.New("signer-key must be provided") + } + if len(o.SerialFile) == 0 { + return errors.New("signer-serial must be provided") + } + + return nil +} + +func (o GetSignerCertOptions) GetSignerCert() (*crypto.CA, error) { + return crypto.GetCA(o.CertFile, o.KeyFile, o.SerialFile) +} diff --git a/pkg/cmd/server/command.go b/pkg/cmd/server/command.go deleted file mode 100644 index 5b944ef476a7..000000000000 --- a/pkg/cmd/server/command.go +++ /dev/null @@ -1,152 +0,0 @@ -package server - -import ( - "errors" - "fmt" - "net" - _ "net/http/pprof" - "strings" - - "github.com/golang/glog" - "github.com/spf13/cobra" - - "github.com/GoogleCloudPlatform/kubernetes/pkg/util" -) - -const longCommandDesc = ` -Start an OpenShift server - -This command helps you launch an OpenShift server. The default mode is all-in-one, which allows -you to run all of the components of an OpenShift system on a server with Docker. Running - - $ openshift start - -will start OpenShift listening on all interfaces, launch an etcd server to store persistent -data, and launch the Kubernetes system components. The server will run in the foreground until -you terminate the process. - -Note: starting OpenShift without passing the --master address will attempt to find the IP -address that will be visible inside running Docker containers. This is not always successful, -so if you have problems tell OpenShift what public address it will be via --master=. - -You may also pass an optional argument to the start command to start OpenShift in one of the -following roles: - - $ openshift start master --nodes= - - Launches the server and control plane for OpenShift. You may pass a list of the node - hostnames you want to use, or create nodes via the REST API or 'openshift kube'. - - $ openshift start node --master= - - Launches a new node and attempts to connect to the master on the provided IP. - -You may also pass --etcd=
to connect to an external etcd server instead of running an -integrated instance, or --kubernetes= and --kubeconfig= to connect to an existing -Kubernetes cluster. -` - -// NewCommandStartServer provides a CLI handler for 'start' command -func NewCommandStartServer(name string) (*cobra.Command, *Config) { - cfg := NewDefaultConfig() - - cmd := &cobra.Command{ - Use: fmt.Sprintf("%s [master|node]", name), - Short: "Launch OpenShift", - Long: longCommandDesc, - Run: func(c *cobra.Command, args []string) { - if err := cfg.Validate(args); err != nil { - glog.Fatal(err) - } - - cfg.Complete(args) - - if err := cfg.Start(args); err != nil { - glog.Fatal(err) - } - }, - } - - flag := cmd.Flags() - - flag.BoolVar(&cfg.WriteConfigOnly, "config-only", false, "Indicates that the command should build the config that would be used to start OpenShift and do nothing else. This is not yet implemented.") - - flag.Var(&cfg.BindAddr, "listen", "The address to listen for connections on (host, host:port, or URL).") - flag.Var(&cfg.MasterAddr, "master", "The master address for use by OpenShift components (host, host:port, or URL). Scheme and port default to the --listen scheme and port.") - flag.Var(&cfg.MasterPublicAddr, "public-master", "The master address for use by public clients, if different (host, host:port, or URL). Defaults to same as --master.") - flag.Var(&cfg.EtcdAddr, "etcd", "The address of the etcd server (host, host:port, or URL). If specified, no built-in etcd will be started.") - flag.Var(&cfg.KubernetesAddr, "kubernetes", "The address of the Kubernetes server (host, host:port, or URL). If specified, no Kubernetes components will be started.") - flag.Var(&cfg.KubernetesPublicAddr, "public-kubernetes", "The Kubernetes server address for use by public clients, if different. (host, host:port, or URL). Defaults to same as --kubernetes.") - flag.Var(&cfg.PortalNet, "portal-net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") - - flag.StringVar(&cfg.ImageTemplate.Format, "images", cfg.ImageTemplate.Format, "When fetching images used by the cluster for important components, use this format on both master and nodes. The latest release will be used by default.") - flag.BoolVar(&cfg.ImageTemplate.Latest, "latest-images", cfg.ImageTemplate.Latest, "If true, attempt to use the latest images for the cluster instead of the latest release.") - - flag.StringVar(&cfg.VolumeDir, "volume-dir", "openshift.local.volumes", "The volume storage directory.") - flag.StringVar(&cfg.EtcdDir, "etcd-dir", "openshift.local.etcd", "The etcd data directory.") - flag.StringVar(&cfg.CertDir, "cert-dir", "openshift.local.certificates", "The certificate data directory.") - - flag.StringVar(&cfg.Hostname, "hostname", cfg.Hostname, "The hostname to identify this node with the master.") - flag.Var(&cfg.NodeList, "nodes", "The hostnames of each node. This currently must be specified up front. Comma delimited list") - flag.Var(&cfg.CORSAllowedOrigins, "cors-allowed-origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. CORS is enabled for localhost, 127.0.0.1, and the asset server by default.") - - flag.StringVar(&cfg.ClientConfigLoadingRules.CommandLinePath, "kubeconfig", "", "Path to the kubeconfig file to use for requests to the Kubernetes API.") - - cfg.Docker.InstallFlags(flag) - - return cmd, cfg -} - -const startMaster = "master" -const startNode = "node" - -func (cfg Config) Validate(args []string) error { - switch len(args) { - case 1: - switch args[0] { - case startMaster: // allowed case - case startNode: // allowed case - default: - return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") - } - case 0: - // do nothing, this starts an all in one - - default: - return errors.New("You may start an OpenShift all-in-one server with no arguments, or pass 'master' or 'node' to run in that role.") - } - - return nil -} - -// Complete takes the args and fills in information for the start config -func (cfg *Config) Complete(args []string) { - cfg.StartMaster = (len(args) == 0) || (args[0] == startMaster) - cfg.StartNode = (len(args) == 0) || (args[0] == startNode) - - if cfg.StartMaster { - // if we've explicitly called out a kube server or a client config, don't start kube in-process - cfg.StartKube = !cfg.KubernetesAddr.Provided && len(cfg.ClientConfigLoadingRules.CommandLinePath) == 0 - // if we've explicitly called out an etcd server, don't start etcd in-process - cfg.StartEtcd = !cfg.EtcdAddr.Provided - } - - // if this is an all-in-one start, be sure to add our hostname to the NodeList if it is not already present - isAllInOne := (len(args) == 0) - if isAllInOne { - nodeList := util.NewStringSet(strings.ToLower(cfg.Hostname)) - // take everything toLower - for _, s := range cfg.NodeList { - nodeList.Insert(strings.ToLower(s)) - } - - cfg.NodeList = nodeList.List() - - // in the all-in-one, default ClusterDNS to the master's address - if url, err := cfg.GetMasterAddress(); err == nil { - if host, _, err := net.SplitHostPort(url.Host); err == nil { - cfg.ClusterDNS = net.ParseIP(host) - } - } - } -} diff --git a/pkg/cmd/server/command_test.go b/pkg/cmd/server/command_test.go deleted file mode 100644 index fc99be068ba4..000000000000 --- a/pkg/cmd/server/command_test.go +++ /dev/null @@ -1,364 +0,0 @@ -package server - -import ( - "strconv" - "strings" - "testing" - - "github.com/spf13/cobra" - - "github.com/GoogleCloudPlatform/kubernetes/pkg/util" -) - -func TestCommandBindingListen(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--listen=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.BindAddr.Set(valueToSet) - - if expectedConfig.BindAddr.String() != actualCfg.BindAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.BindAddr.String(), actualCfg.BindAddr.String()) - } -} - -func TestCommandBindingMaster(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--master=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.MasterAddr.Set(valueToSet) - - if expectedConfig.MasterAddr.String() != actualCfg.MasterAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.MasterAddr.String(), actualCfg.MasterAddr.String()) - } -} - -func TestCommandBindingMasterPublic(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--public-master=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.MasterPublicAddr.Set(valueToSet) - - if expectedConfig.MasterPublicAddr.String() != actualCfg.MasterPublicAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.MasterPublicAddr.String(), actualCfg.MasterPublicAddr.String()) - } -} - -func TestCommandBindingEtcd(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--etcd=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.EtcdAddr.Set(valueToSet) - - if expectedConfig.EtcdAddr.String() != actualCfg.EtcdAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.EtcdAddr.String(), actualCfg.EtcdAddr.String()) - } -} - -func TestCommandBindingKubernetes(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--kubernetes=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.KubernetesAddr.Set(valueToSet) - - if expectedConfig.KubernetesAddr.String() != actualCfg.KubernetesAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.KubernetesAddr.String(), actualCfg.KubernetesAddr.String()) - } -} - -func TestCommandBindingKubernetesPublic(t *testing.T) { - valueToSet := "http://example.org:9123" - actualCfg := executeCommand([]string{"--public-kubernetes=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.KubernetesPublicAddr.Set(valueToSet) - - if expectedConfig.KubernetesPublicAddr.String() != actualCfg.KubernetesPublicAddr.String() { - t.Errorf("expected %v, got %v", expectedConfig.KubernetesPublicAddr.String(), actualCfg.KubernetesPublicAddr.String()) - } -} - -func TestCommandBindingPortalNet(t *testing.T) { - valueToSet := "192.168.0.0/16" - actualCfg := executeCommand([]string{"--portal-net=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.PortalNet.Set(valueToSet) - - if expectedConfig.PortalNet.String() != actualCfg.PortalNet.String() { - t.Errorf("expected %v, got %v", expectedConfig.PortalNet.String(), actualCfg.PortalNet.String()) - } -} - -func TestCommandBindingImageTemplateFormat(t *testing.T) { - valueToSet := "some-format-string" - actualCfg := executeCommand([]string{"--images=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.ImageTemplate.Format = valueToSet - - if expectedConfig.ImageTemplate.Format != actualCfg.ImageTemplate.Format { - t.Errorf("expected %v, got %v", expectedConfig.ImageTemplate.Format, actualCfg.ImageTemplate.Format) - } -} - -func TestCommandBindingImageLatest(t *testing.T) { - expectedConfig := NewDefaultConfig() - - valueToSet := strconv.FormatBool(!expectedConfig.ImageTemplate.Latest) - actualCfg := executeCommand([]string{"--latest-images=" + valueToSet}) - - expectedConfig.ImageTemplate.Latest = !expectedConfig.ImageTemplate.Latest - - if expectedConfig.ImageTemplate.Latest != actualCfg.ImageTemplate.Latest { - t.Errorf("expected %v, got %v", expectedConfig.ImageTemplate.Latest, actualCfg.ImageTemplate.Latest) - } -} - -func TestCommandBindingVolumeDir(t *testing.T) { - valueToSet := "some-string" - actualCfg := executeCommand([]string{"--volume-dir=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.VolumeDir = valueToSet - - if expectedConfig.VolumeDir != actualCfg.VolumeDir { - t.Errorf("expected %v, got %v", expectedConfig.VolumeDir, actualCfg.VolumeDir) - } -} - -func TestCommandBindingEtcdDir(t *testing.T) { - valueToSet := "some-string" - actualCfg := executeCommand([]string{"--etcd-dir=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.EtcdDir = valueToSet - - if expectedConfig.EtcdDir != actualCfg.EtcdDir { - t.Errorf("expected %v, got %v", expectedConfig.EtcdDir, actualCfg.EtcdDir) - } -} - -func TestCommandBindingCertDir(t *testing.T) { - valueToSet := "some-string" - actualCfg := executeCommand([]string{"--cert-dir=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.CertDir = valueToSet - - if expectedConfig.CertDir != actualCfg.CertDir { - t.Errorf("expected %v, got %v", expectedConfig.CertDir, actualCfg.CertDir) - } -} - -func TestCommandBindingHostname(t *testing.T) { - valueToSet := "some-string" - actualCfg := executeCommand([]string{"--hostname=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.Hostname = valueToSet - - if expectedConfig.Hostname != actualCfg.Hostname { - t.Errorf("expected %v, got %v", expectedConfig.Hostname, actualCfg.Hostname) - } -} - -// AllInOne always adds the default hostname -func TestCommandBindingNodesForAllInOneAppend(t *testing.T) { - valueToSet := "first,second,third" - actualCfg := executeCommand([]string{"--nodes=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - - stringList := util.StringList{} - stringList.Set(valueToSet + "," + strings.ToLower(expectedConfig.Hostname)) - expectedConfig.NodeList.Set(strings.Join(util.NewStringSet(stringList...).List(), ",")) - - if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { - t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) - } -} - -// AllInOne always adds the default hostname -func TestCommandBindingNodesForAllInOneAppendNoDupes(t *testing.T) { - - valueToSet := "first,localhost,second,third" - actualCfg := executeCommand([]string{"--nodes=" + valueToSet, "--hostname=LOCALHOST"}) - - expectedConfig := NewDefaultConfig() - expectedConfig.NodeList.Set(valueToSet) - - util.NewStringSet() - - if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { - t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) - } -} - -// AllInOne always adds the default hostname -func TestCommandBindingNodesDefaultingAllInOne(t *testing.T) { - actualCfg := executeCommand([]string{}) - - expectedConfig := NewDefaultConfig() - expectedConfig.NodeList.Set(strings.ToLower(expectedConfig.Hostname)) - - if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { - t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) - } -} - -// explicit start master never modifies the NodeList -func TestCommandBindingNodesForMaster(t *testing.T) { - valueToSet := "first,second,third" - actualCfg := executeCommand([]string{"master", "--nodes=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.NodeList.Set(valueToSet) - - if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { - t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) - } -} - -// explicit start master never modifies the NodeList -func TestCommandBindingNodesDefaultingMaster(t *testing.T) { - actualCfg := executeCommand([]string{"master"}) - - expectedConfig := NewDefaultConfig() - expectedConfig.NodeList.Set("") - - if expectedConfig.NodeList.String() != actualCfg.NodeList.String() { - t.Errorf("expected %v, got %v", expectedConfig.NodeList, actualCfg.NodeList) - } -} - -func TestCommandBindingCors(t *testing.T) { - valueToSet := "first,second,third" - actualCfg := executeCommand([]string{"--cors-allowed-origins=" + valueToSet}) - - expectedConfig := NewDefaultConfig() - expectedConfig.CORSAllowedOrigins.Set(valueToSet) - - if expectedConfig.CORSAllowedOrigins.String() != actualCfg.CORSAllowedOrigins.String() { - t.Errorf("expected %v, got %v", expectedConfig.CORSAllowedOrigins, actualCfg.CORSAllowedOrigins) - } -} - -func TestCommandCompletionNode(t *testing.T) { - commandCompletionTest{ - args: []string{"node"}, - - StartNode: true, - }.run(t) -} - -func TestCommandCompletionMaster(t *testing.T) { - commandCompletionTest{ - args: []string{"master"}, - - StartMaster: true, - StartKube: true, - StartEtcd: true, - }.run(t) -} -func TestCommandCompletionMasterExternalKubernetes(t *testing.T) { - commandCompletionTest{ - args: []string{"master", "--kubernetes=foo"}, - - StartMaster: true, - StartKube: false, - StartEtcd: true, - }.run(t) -} -func TestCommandCompletionMasterExternalKubernetesConfig(t *testing.T) { - commandCompletionTest{ - args: []string{"master", "--kubeconfig=foo"}, - - StartMaster: true, - StartKube: false, - StartEtcd: true, - }.run(t) -} - -func TestCommandCompletionAllInOne(t *testing.T) { - commandCompletionTest{ - StartNode: true, - StartMaster: true, - StartKube: true, - StartEtcd: true, - }.run(t) -} -func TestCommandCompletionAllInOneExternalKubernetes(t *testing.T) { - commandCompletionTest{ - args: []string{"--kubernetes=foo"}, - - StartNode: true, - StartMaster: true, - StartKube: false, - StartEtcd: true, - }.run(t) -} -func TestCommandCompletionAllInOneExternalKubernetesConfig(t *testing.T) { - commandCompletionTest{ - args: []string{"--kubeconfig=foo"}, - - StartNode: true, - StartMaster: true, - StartKube: false, - StartEtcd: true, - }.run(t) -} - -type commandCompletionTest struct { - args []string - - StartNode bool - StartMaster bool - StartKube bool - StartEtcd bool -} - -func executeCommand(args []string) *Config { - argsToUse := make([]string, 0, 1+len(args)) - argsToUse = append(argsToUse, "start") - argsToUse = append(argsToUse, args...) - argsToUse = append(argsToUse, "--config-only") - - root := &cobra.Command{ - Use: "openshift", - Short: "test", - Long: "", - Run: func(c *cobra.Command, args []string) { - c.Help() - }, - } - - openshiftStartCommand, cfg := NewCommandStartServer("start") - root.AddCommand(openshiftStartCommand) - root.SetArgs(argsToUse) - root.Execute() - - return cfg -} - -func (test commandCompletionTest) run(t *testing.T) { - actualCfg := executeCommand(test.args) - - if test.StartNode != actualCfg.StartNode { - t.Errorf("expected %v, got %v", test.StartNode, actualCfg.StartNode) - } - if test.StartMaster != actualCfg.StartMaster { - t.Errorf("expected %v, got %v", test.StartMaster, actualCfg.StartMaster) - } - if test.StartKube != actualCfg.StartKube { - t.Errorf("expected %v, got %v", test.StartKube, actualCfg.StartKube) - } - if test.StartEtcd != actualCfg.StartEtcd { - t.Errorf("expected %v, got %v", test.StartEtcd, actualCfg.StartEtcd) - } - -} diff --git a/pkg/cmd/server/config.go b/pkg/cmd/server/config.go deleted file mode 100644 index abe9634d5064..000000000000 --- a/pkg/cmd/server/config.go +++ /dev/null @@ -1,303 +0,0 @@ -package server - -import ( - "fmt" - "net" - "net/url" - "os/exec" - "strconv" - "strings" - "time" - - etcdclient "github.com/coreos/go-etcd/etcd" - "github.com/golang/glog" - - "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" - "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" - kutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util" - - "github.com/openshift/origin/pkg/api/latest" - "github.com/openshift/origin/pkg/cmd/flagtypes" - "github.com/openshift/origin/pkg/cmd/util" - "github.com/openshift/origin/pkg/cmd/util/docker" - "github.com/openshift/origin/pkg/cmd/util/variable" -) - -// Config is a struct that the command stores flag values into. -type Config struct { - Docker *docker.Helper - - WriteConfigOnly bool - - StartNode bool - StartMaster bool - StartKube bool - StartEtcd bool - - MasterAddr flagtypes.Addr - BindAddr flagtypes.Addr - EtcdAddr flagtypes.Addr - KubernetesAddr flagtypes.Addr - PortalNet flagtypes.IPNet - DNSBindAddr flagtypes.Addr - // addresses for external clients - MasterPublicAddr flagtypes.Addr - KubernetesPublicAddr flagtypes.Addr - // addresses for asset server - AssetBindAddr flagtypes.Addr - AssetPublicAddr flagtypes.Addr - - ImageTemplate variable.ImageTemplate - - Hostname string - VolumeDir string - - EtcdDir string - - CertDir string - - ClusterDNS net.IP - - StorageVersion string - - NodeList kutil.StringList - - // ClientConfig is used when connecting to Kubernetes from the master, or - // when connecting to the master from a detached node. If StartKube is true, - // this value is not used. - ClientConfig clientcmd.ClientConfig - // ClientConfigLoadingRules is the ruleset used to load the client config. - // Only the CommandLinePath is expected to be used. - ClientConfigLoadingRules clientcmd.ClientConfigLoadingRules - - CORSAllowedOrigins kutil.StringList -} - -func NewDefaultConfig() *Config { - hostname, err := defaultHostname() - if err != nil { - hostname = "localhost" - glog.Warningf("Unable to lookup hostname, using %q: %v", hostname, err) - } - - // TODO: secure etcd by default - - config := &Config{ - Docker: docker.NewHelper(), - - MasterAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - BindAddr: flagtypes.Addr{Value: "0.0.0.0:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - EtcdAddr: flagtypes.Addr{Value: "0.0.0.0:4001", DefaultScheme: "http", DefaultPort: 4001}.Default(), - KubernetesAddr: flagtypes.Addr{DefaultScheme: "https", DefaultPort: 8443}.Default(), - PortalNet: flagtypes.DefaultIPNet("172.30.17.0/24"), - MasterPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - KubernetesPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), - AssetPublicAddr: flagtypes.Addr{Value: "localhost:8444", DefaultScheme: "https", DefaultPort: 8444, AllowPrefix: true}.Default(), - AssetBindAddr: flagtypes.Addr{Value: "0.0.0.0:8444", DefaultScheme: "https", DefaultPort: 8444, AllowPrefix: true}.Default(), - - ImageTemplate: variable.NewDefaultImageTemplate(), - - Hostname: hostname, - } - - // TODO: allow DNS binding to be disabled. - config.DNSBindAddr = flagtypes.Addr{Value: config.BindAddr.Host, DefaultPort: 53}.Default() - - config.ClientConfig = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&config.ClientConfigLoadingRules, &clientcmd.ConfigOverrides{}) - - return config -} - -// GetMasterAddress checks for an unset master address and then attempts to use the first -// public IPv4 non-loopback address registered on this host. -// TODO: make me IPv6 safe -func (cfg Config) GetMasterAddress() (*url.URL, error) { - if cfg.MasterAddr.Provided { - return cfg.MasterAddr.URL, nil - } - - // If the user specifies a bind address, and the master is not provided, use the bind port by default - port := cfg.MasterAddr.Port - if cfg.BindAddr.Provided { - port = cfg.BindAddr.Port - } - - // If the user specifies a bind address, and the master is not provided, use the bind scheme by default - scheme := cfg.MasterAddr.URL.Scheme - if cfg.BindAddr.Provided { - scheme = cfg.BindAddr.URL.Scheme - } - - // use the default ip address for the system - addr := "" - if ip, err := util.DefaultLocalIP4(); err == nil { - addr = ip.String() - } else if err == util.ErrorNoDefaultIP { - addr = "127.0.0.1" - } else if err != nil { - return nil, fmt.Errorf("Unable to find a public IP address: %v", err) - } - - masterAddr := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port)) - return url.Parse(masterAddr) -} - -func (cfg Config) GetMasterPublicAddress() (*url.URL, error) { - if cfg.MasterPublicAddr.Provided { - return cfg.MasterPublicAddr.URL, nil - } - - return cfg.GetMasterAddress() -} - -func (cfg Config) GetEtcdBindAddress() string { - // Derive the etcd bind address by using the bind address and the default etcd port - return net.JoinHostPort(cfg.BindAddr.Host, strconv.Itoa(cfg.EtcdAddr.DefaultPort)) -} - -func (cfg Config) GetEtcdPeerBindAddress() string { - // Derive the etcd peer address by using the bind address and the default etcd peering port - return net.JoinHostPort(cfg.BindAddr.Host, "7001") -} - -func (cfg Config) GetEtcdAddress() (*url.URL, error) { - if cfg.EtcdAddr.Provided { - return cfg.EtcdAddr.URL, nil - } - - // Etcd should be reachable on the same address that the master is (for simplicity) - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - - etcdAddr := net.JoinHostPort(getHost(*masterAddr), strconv.Itoa(cfg.EtcdAddr.DefaultPort)) - return url.Parse(cfg.EtcdAddr.DefaultScheme + "://" + etcdAddr) -} - -func (cfg Config) GetExternalKubernetesClientConfig() (*client.Config, bool, error) { - if len(cfg.ClientConfigLoadingRules.CommandLinePath) == 0 || cfg.ClientConfig == nil { - return nil, false, nil - } - clientConfig, err := cfg.ClientConfig.ClientConfig() - if err != nil { - return nil, false, err - } - return clientConfig, true, nil -} - -func (cfg Config) GetKubernetesAddress() (*url.URL, error) { - if cfg.KubernetesAddr.Provided { - return cfg.KubernetesAddr.URL, nil - } - - config, ok, err := cfg.GetExternalKubernetesClientConfig() - if err != nil { - return nil, err - } - if ok && len(config.Host) > 0 { - return url.Parse(config.Host) - } - - return cfg.GetMasterAddress() -} - -func (cfg Config) GetKubernetesPublicAddress() (*url.URL, error) { - if cfg.KubernetesPublicAddr.Provided { - return cfg.KubernetesPublicAddr.URL, nil - } - if cfg.KubernetesAddr.Provided { - return cfg.KubernetesAddr.URL, nil - } - config, ok, err := cfg.GetExternalKubernetesClientConfig() - if err != nil { - return nil, err - } - if ok && len(config.Host) > 0 { - return url.Parse(config.Host) - } - - return cfg.GetMasterPublicAddress() -} - -func (cfg Config) GetAssetPublicAddress() (*url.URL, error) { - if cfg.AssetPublicAddr.Provided { - return cfg.AssetPublicAddr.URL, nil - } - // Derive the asset public address by incrementing the master public address port by 1 - // TODO: derive the scheme/port from the asset bind scheme/port once that is settable via the command line - t, err := cfg.GetMasterPublicAddress() - if err != nil { - return nil, err - } - assetPublicAddr := *t - assetPublicAddr.Host = net.JoinHostPort(getHost(assetPublicAddr), strconv.Itoa(getPort(assetPublicAddr)+1)) - - return &assetPublicAddr, nil -} - -func (cfg Config) GetAssetBindAddress() string { - if cfg.AssetBindAddr.Provided { - return cfg.AssetBindAddr.URL.Host - } - // Derive the asset bind address by incrementing the master bind address port by 1 - return net.JoinHostPort(cfg.BindAddr.Host, strconv.Itoa(cfg.BindAddr.Port+1)) -} - -// getAndTestEtcdClient creates an etcd client based on the provided config and waits -// until etcd server is reachable. It errors out and exits if the server cannot -// be reached for a certain amount of time. -func (cfg Config) getAndTestEtcdClient() (*etcdclient.Client, error) { - address, err := cfg.GetEtcdAddress() - if err != nil { - return nil, err - } - etcdServers := []string{address.String()} - etcdClient := etcdclient.NewClient(etcdServers) - - for i := 0; ; i++ { - // TODO: make sure this works with etcd2 (root key may not exist) - _, err := etcdClient.Get("/", false, false) - if err == nil || tools.IsEtcdNotFound(err) { - break - } - if i > 100 { - return nil, fmt.Errorf("Could not reach etcd: %v", err) - } - time.Sleep(50 * time.Millisecond) - } - - return etcdClient, nil -} - -// newOpenShiftEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version -// is incorrect. -func (cfg Config) newOpenShiftEtcdHelper() (helper tools.EtcdHelper, err error) { - // Connect and setup etcd interfaces - client, err := cfg.getAndTestEtcdClient() - if err != nil { - return tools.EtcdHelper{}, err - } - - version := cfg.StorageVersion - if len(version) == 0 { - version = latest.Version - } - interfaces, err := latest.InterfacesFor(version) - if err != nil { - return helper, err - } - return tools.EtcdHelper{client, interfaces.Codec, tools.RuntimeVersionAdapter{interfaces.MetadataAccessor}}, nil -} - -// defaultHostname returns the default hostname for this system. -func defaultHostname() (string, error) { - // Note: We use exec here instead of os.Hostname() because we - // want the FQDN, and this is the easiest way to get it. - fqdn, err := exec.Command("hostname", "-f").Output() - if err != nil { - return "", fmt.Errorf("Couldn't determine hostname: %v", err) - } - return strings.TrimSpace(string(fqdn)), nil -} diff --git a/pkg/cmd/server/crypto/crypto.go b/pkg/cmd/server/crypto/crypto.go index 343b9d0c2b2f..102b715e5243 100644 --- a/pkg/cmd/server/crypto/crypto.go +++ b/pkg/cmd/server/crypto/crypto.go @@ -20,92 +20,102 @@ import ( "strconv" "time" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" - kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - clientcmd "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" - clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" "github.com/golang/glog" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + "github.com/openshift/origin/pkg/auth/authenticator/request/x509request" ) -func filenamesFromDir(dir string) (string, string, string) { - return filepath.Join(dir, "root.crt"), filepath.Join(dir, "cert.crt"), filepath.Join(dir, "key.key") -} - type TLSCertificateConfig struct { - CAFile string - CertFile string - KeyFile string - - Roots []*x509.Certificate Certs []*x509.Certificate Key crypto.PrivateKey } -func (c *TLSCertificateConfig) writeDir(dir string) error { - c.CAFile, c.CertFile, c.KeyFile = filenamesFromDir(dir) - - // mkdir - if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { - return err - } +type TLSCARoots struct { + Roots []*x509.Certificate +} - // write certs and keys - if err := writeCertificates(c.CAFile, c.Roots...); err != nil { +func (c *TLSCertificateConfig) writeCertConfig(certFile, keyFile string) error { + if err := writeCertificates(certFile, c.Certs...); err != nil { return err } - if err := writeCertificates(c.CertFile, c.Certs...); err != nil { + if err := writeKeyFile(keyFile, c.Key); err != nil { return err } - if err := writeKeyFile(c.KeyFile, c.Key); err != nil { + return nil +} +func (c *TLSCARoots) writeCARoots(rootFile string) error { + if err := writeCertificates(rootFile, c.Roots...); err != nil { return err } return nil } -func newTLSCertificateConfig(dir string) (*TLSCertificateConfig, error) { - caFile, certFile, keyFile := filenamesFromDir(dir) - config := &TLSCertificateConfig{ - CAFile: caFile, - CertFile: certFile, - KeyFile: keyFile, +func GetTLSCARoots(caFile string) (*TLSCARoots, error) { + if len(caFile) == 0 { + return nil, errors.New("caFile missing") } - if caFile != "" { - caPEMBlock, err := ioutil.ReadFile(caFile) - if err != nil { - return nil, err - } - config.Roots, err = certsFromPEM(caPEMBlock) - if err != nil { - return nil, fmt.Errorf("Error reading %s: %s", caFile, err) - } + caPEMBlock, err := ioutil.ReadFile(caFile) + if err != nil { + return nil, err + } + roots, err := certsFromPEM(caPEMBlock) + if err != nil { + return nil, fmt.Errorf("Error reading %s: %s", caFile, err) } - if certFile != "" { - certPEMBlock, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, err - } - config.Certs, err = certsFromPEM(certPEMBlock) - if err != nil { - return nil, fmt.Errorf("Error reading %s: %s", caFile, err) - } + return &TLSCARoots{roots}, nil +} - if keyFile != "" { - keyPEMBlock, err := ioutil.ReadFile(keyFile) - if err != nil { - return nil, err - } - keyPairCert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) - if err != nil { - return nil, err - } - config.Key = keyPairCert.PrivateKey - } +func GetTLSCertificateConfig(certFile, keyFile string) (*TLSCertificateConfig, error) { + if len(certFile) == 0 { + return nil, errors.New("certFile missing") + } + if len(keyFile) == 0 { + return nil, errors.New("keyFile missing") + } + + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return nil, err + } + certs, err := certsFromPEM(certPEMBlock) + if err != nil { + return nil, fmt.Errorf("Error reading %s: %s", certFile, err) + } + + keyPEMBlock, err := ioutil.ReadFile(keyFile) + if err != nil { + return nil, err + } + keyPairCert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + return nil, err + } + key := keyPairCert.PrivateKey + + return &TLSCertificateConfig{certs, key}, nil +} + +func CertPoolFromFile(filename string) (*x509.CertPool, error) { + pemBlock, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + certs, err := certsFromPEM(pemBlock) + if err != nil { + return nil, fmt.Errorf("Error reading %s: %s", filename, err) } - return config, nil + roots := x509.NewCertPool() + for _, root := range certs { + roots.AddCert(root) + } + + return roots, nil } func certsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) { @@ -145,55 +155,31 @@ var ( ) type CA struct { - Dir string SerialFile string Serial int64 Config *TLSCertificateConfig } -// InitCA ensures a certificate authority structure exists in the given directory, creating it if necessary: -// / -// ca/ -// root.crt - Root certificate bundle. -// cert.crt - Signing certificate -// key.key - Private key -// serial.txt - Stores the highest serial number generated by this CA -func InitCA(dir string, name string) (*CA, error) { - caDir := filepath.Join(dir, "ca") +func EnsureCA(certFile, keyFile, serialFile, name string) (*CA, error) { + if ca, err := GetCA(certFile, keyFile, serialFile); err == nil { + return ca, nil + } + + return MakeCA(certFile, keyFile, serialFile, name) +} - caConfig, err := newTLSCertificateConfig(caDir) +func GetCA(certFile, keyFile, serialFile string) (*CA, error) { + caConfig, err := GetTLSCertificateConfig(certFile, keyFile) if err != nil { - glog.V(2).Infof("Generating new CA in %s", caDir) - // Create CA cert - rootcaPublicKey, rootcaPrivateKey, err := NewKeyPair() - if err != nil { - return nil, err - } - rootcaTemplate, err := newSigningCertificateTemplate(pkix.Name{CommonName: name}) - if err != nil { - return nil, err - } - rootcaCert, err := signCertificate(rootcaTemplate, rootcaPublicKey, rootcaTemplate, rootcaPrivateKey) - if err != nil { - return nil, err - } - caConfig = &TLSCertificateConfig{ - Roots: []*x509.Certificate{rootcaCert}, - Certs: []*x509.Certificate{rootcaCert}, - Key: rootcaPrivateKey, - } - if err := caConfig.writeDir(caDir); err != nil { - return nil, err - } - } else { - glog.V(2).Infof("Using existing CA certificate in %s", caDir) + return nil, err } // read serial file var serial int64 - serialFile := filepath.Join(caDir, "serial.txt") if serialData, err := ioutil.ReadFile(serialFile); err == nil { serial, _ = strconv.ParseInt(string(serialData), 10, 64) + } else { + return nil, err } if serial < 1 { serial = 1 @@ -202,159 +188,127 @@ func InitCA(dir string, name string) (*CA, error) { return &CA{ Serial: serial, SerialFile: serialFile, - Dir: dir, Config: caConfig, }, nil } -// MakeServerCert creates a folder containing certificates for the given server: -// / -// / -// root.crt - Root certificate bundle. -// cert.crt - Server certificate -// key.key - Private key -// The generated certificate has the following attributes: -// CommonName: hostnames[0] -// DNSNames subjectAltNames containing all specified hostnames -// IPAddresses subjectAltNames containing all specified hostnames which are IP addresses -// ExtKeyUsage: ExtKeyUsageServerAuth -func (ca *CA) MakeServerCert(name string, hostnames []string) (*TLSCertificateConfig, error) { - serverDir := filepath.Join(ca.Dir, name) - - server, err := newTLSCertificateConfig(serverDir) - if err == nil { - cert := server.Certs[0] - ips, dns := IPAddressesDNSNames(hostnames) - missingIps := ipsNotInSlice(ips, cert.IPAddresses) - missingDns := stringsNotInSlice(dns, cert.DNSNames) - if len(missingIps) == 0 && len(missingDns) == 0 { - glog.V(2).Infof("Using existing server certificate in %s", serverDir) - return server, nil - } +func MakeCA(certFile, keyFile, serialFile, name string) (*CA, error) { + glog.V(2).Infof("Generating new CA for %s cert, and key in %s, %s", name, certFile, keyFile) + // Create CA cert + rootcaPublicKey, rootcaPrivateKey, err := NewKeyPair() + if err != nil { + return nil, err + } + rootcaTemplate, err := newSigningCertificateTemplate(pkix.Name{CommonName: name}) + if err != nil { + return nil, err + } + rootcaCert, err := signCertificate(rootcaTemplate, rootcaPublicKey, rootcaTemplate, rootcaPrivateKey) + if err != nil { + return nil, err + } + caConfig := &TLSCertificateConfig{ + Certs: []*x509.Certificate{rootcaCert}, + Key: rootcaPrivateKey, + } + if err := caConfig.writeCertConfig(certFile, keyFile); err != nil { + return nil, err + } + + if err := ioutil.WriteFile(serialFile, []byte("0"), 0644); err != nil { + return nil, err + } + + return &CA{ + Serial: 0, + SerialFile: serialFile, + Config: caConfig, + }, nil +} - glog.V(2).Infof("Existing server certificate in %s was missing some hostnames (%v) or IP addresses (%v)", serverDir, missingDns, missingIps) +func (ca *CA) EnsureServerCert(certFile, keyFile string, hostnames util.StringSet) (*TLSCertificateConfig, error) { + certConfig, err := GetServerCert(certFile, keyFile, hostnames) + if err != nil { + return ca.MakeServerCert(certFile, keyFile, hostnames) } - glog.V(2).Infof("Generating server certificate in %s", serverDir) + return certConfig, nil +} + +func GetServerCert(certFile, keyFile string, hostnames util.StringSet) (*TLSCertificateConfig, error) { + server, err := GetTLSCertificateConfig(certFile, keyFile) + if err != nil { + return nil, err + } + + cert := server.Certs[0] + ips, dns := IPAddressesDNSNames(hostnames.List()) + missingIps := ipsNotInSlice(ips, cert.IPAddresses) + missingDns := stringsNotInSlice(dns, cert.DNSNames) + if len(missingIps) == 0 && len(missingDns) == 0 { + glog.V(2).Infof("Found existing server certificate in %s", certFile) + return server, nil + } + + return nil, fmt.Errorf("Existing server certificate in %s was missing some hostnames (%v) or IP addresses (%v).", certFile, missingDns, missingIps) +} + +func (ca *CA) MakeServerCert(certFile, keyFile string, hostnames util.StringSet) (*TLSCertificateConfig, error) { + glog.V(2).Infof("Generating server certificate in %s, key in %s", certFile, keyFile) + serverPublicKey, serverPrivateKey, _ := NewKeyPair() - serverTemplate, _ := newServerCertificateTemplate(pkix.Name{CommonName: hostnames[0]}, hostnames) + serverTemplate, _ := newServerCertificateTemplate(pkix.Name{CommonName: hostnames.List()[0]}, hostnames.List()) serverCrt, _ := ca.signCertificate(serverTemplate, serverPublicKey) - server = &TLSCertificateConfig{ - Roots: ca.Config.Roots, + server := &TLSCertificateConfig{ Certs: append([]*x509.Certificate{serverCrt}, ca.Config.Certs...), Key: serverPrivateKey, } - if err := server.writeDir(serverDir); err != nil { + if err := server.writeCertConfig(certFile, keyFile); err != nil { return server, err } return server, nil } -// MakeClientConfig creates a folder containing certificates for the given client: -// / -// / -// root.crt - Root certificate bundle. -// cert.crt - Client certificate -// key.key - Private key -// .kubeconfig - baseKubeconfig with root.crt added to all clusters, and client user added to all contexts -// The generated certificate has the following attributes: -// Subject: -// SerialNumber: user.GetUID() -// CommonName: user.GetName() -// Organization: user.GetGroups() -// ExtKeyUsage: ExtKeyUsageClientAuth -func (ca *CA) MakeClientConfig(clientId string, u user.Info, baseKubeconfig clientcmdapi.Config) (kclient.Config, error) { - var ( - client kclient.Config - err error - caFile, certFile, keyFile string - caData, certData, keyData []byte - ) - - // Ensure the folder exists - clientDir := filepath.Join(ca.Dir, clientId) - if err := os.MkdirAll(clientDir, os.FileMode(0755)); err != nil { - return client, err - } - - caFile, certFile, keyFile = filenamesFromDir(clientDir) - if err == nil { - caData, err = ioutil.ReadFile(caFile) - } - if err == nil { - certData, err = ioutil.ReadFile(certFile) - } - if err == nil { - keyData, err = ioutil.ReadFile(keyFile) - } - - if err == nil { - glog.V(2).Infof("Using existing client certificates in %s", clientDir) - } else { - glog.V(2).Infof("Generating client certificates in %s", clientDir) - - clientPublicKey, clientPrivateKey, _ := NewKeyPair() - clientTemplate, _ := newClientCertificateTemplate(x509request.UserToSubject(u)) - clientCrt, _ := ca.signCertificate(clientTemplate, clientPublicKey) +func (ca *CA) EnsureClientCertificate(certFile, keyFile string, u user.Info) (*TLSCertificateConfig, error) { + certConfig, err := GetTLSCertificateConfig(certFile, keyFile) + if err != nil { + return ca.MakeClientCertificate(certFile, keyFile, u) + } - caData, err = encodeCertificates(ca.Config.Roots...) - if err != nil { - return client, err - } - certData, err = encodeCertificates(clientCrt) - if err != nil { - return client, err - } - keyData, err = encodeKey(clientPrivateKey) - if err != nil { - return client, err - } + return certConfig, nil +} - // write files - if err = ioutil.WriteFile(caFile, caData, os.FileMode(0644)); err != nil { - return client, err - } - if err = ioutil.WriteFile(certFile, certData, os.FileMode(0644)); err != nil { - return client, err - } - if err = ioutil.WriteFile(keyFile, keyData, os.FileMode(0600)); err != nil { - return client, err - } +func (ca *CA) MakeClientCertificate(certFile, keyFile string, u user.Info) (*TLSCertificateConfig, error) { + glog.V(2).Infof("Generating client cert in %s and key in %s", certFile, keyFile) + // ensure parent dirs + if err := os.MkdirAll(filepath.Dir(certFile), os.FileMode(0755)); err != nil { + return nil, err } - - // Set the generated certs on a user - baseKubeconfig.AuthInfos = map[string]clientcmdapi.AuthInfo{ - clientId: { - ClientCertificateData: certData, - ClientKeyData: keyData, - }, + if err := os.MkdirAll(filepath.Dir(keyFile), os.FileMode(0755)); err != nil { + return nil, err } - // Set the ca certs on all clusters - for clusterName, cluster := range baseKubeconfig.Clusters { - cluster.CertificateAuthorityData = caData - baseKubeconfig.Clusters[clusterName] = cluster - } + clientPublicKey, clientPrivateKey, _ := NewKeyPair() + clientTemplate, _ := newClientCertificateTemplate(x509request.UserToSubject(u)) + clientCrt, _ := ca.signCertificate(clientTemplate, clientPublicKey) - // Use the new user in all contexts - for contextName, context := range baseKubeconfig.Contexts { - context.AuthInfo = clientId - baseKubeconfig.Contexts[contextName] = context + certData, err := encodeCertificates(clientCrt) + if err != nil { + return nil, err } - - if builtClient, err := clientcmd.NewDefaultClientConfig(baseKubeconfig, &clientcmd.ConfigOverrides{}).ClientConfig(); err != nil { - return client, err - } else { - client = *builtClient + keyData, err := encodeKey(clientPrivateKey) + if err != nil { + return nil, err } - kubeConfigFile := filepath.Join(clientDir, ".kubeconfig") - glog.V(2).Infof("Writing client config in %s", kubeConfigFile) - if err := clientcmd.WriteToFile(baseKubeconfig, kubeConfigFile); err != nil { - return client, err + if err = ioutil.WriteFile(certFile, certData, os.FileMode(0644)); err != nil { + return nil, err + } + if err = ioutil.WriteFile(keyFile, keyData, os.FileMode(0600)); err != nil { + return nil, err } - return client, nil + return GetTLSCertificateConfig(certFile, keyFile) } func (ca *CA) signCertificate(template *x509.Certificate, requestKey crypto.PublicKey) (*x509.Certificate, error) { @@ -518,6 +472,11 @@ func encodeKey(key crypto.PrivateKey) ([]byte, error) { } func writeCertificates(path string, certs ...*x509.Certificate) error { + // ensure parent dir + if err := os.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil { + return err + } + bytes, err := encodeCertificates(certs...) if err != nil { return err @@ -525,6 +484,11 @@ func writeCertificates(path string, certs ...*x509.Certificate) error { return ioutil.WriteFile(path, bytes, os.FileMode(0644)) } func writeKeyFile(path string, key crypto.PrivateKey) error { + // ensure parent dir + if err := os.MkdirAll(filepath.Dir(path), os.FileMode(0755)); err != nil { + return err + } + b, err := encodeKey(key) if err != nil { return err diff --git a/pkg/cmd/server/doc.go b/pkg/cmd/server/doc.go deleted file mode 100644 index 0bd9d55b7705..000000000000 --- a/pkg/cmd/server/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package server contains the main command for launching an OpenShift server. Subpackages -// contain the specific components that the server may need. -package server diff --git a/pkg/cmd/server/etcd/etcd.go b/pkg/cmd/server/etcd/etcd.go index 9d98f1114000..f52af1783c73 100644 --- a/pkg/cmd/server/etcd/etcd.go +++ b/pkg/cmd/server/etcd/etcd.go @@ -1,12 +1,18 @@ package etcd import ( + "fmt" "time" - "github.com/GoogleCloudPlatform/kubernetes/pkg/util" etcdconfig "github.com/coreos/etcd/config" "github.com/coreos/etcd/etcd" + etcdclient "github.com/coreos/go-etcd/etcd" "github.com/golang/glog" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/api/latest" ) // Config is an object that can run an etcd server @@ -34,3 +40,42 @@ func (c *Config) Run() { }, 500*time.Millisecond) <-server.ReadyNotify() } + +// getAndTestEtcdClient creates an etcd client based on the provided config and waits +// until etcd server is reachable. It errors out and exits if the server cannot +// be reached for a certain amount of time. +func GetAndTestEtcdClient(etcdURL string) (*etcdclient.Client, error) { + etcdServers := []string{etcdURL} + etcdClient := etcdclient.NewClient(etcdServers) + + for i := 0; ; i++ { + // TODO: make sure this works with etcd2 (root key may not exist) + _, err := etcdClient.Get("/", false, false) + if err == nil || tools.IsEtcdNotFound(err) { + break + } + if i > 100 { + return nil, fmt.Errorf("Could not reach etcd: %v", err) + } + time.Sleep(50 * time.Millisecond) + } + + return etcdClient, nil +} + +// newOpenShiftEtcdHelper returns an EtcdHelper for the provided arguments or an error if the version +// is incorrect. +func NewOpenShiftEtcdHelper(etcdURL string) (helper tools.EtcdHelper, err error) { + // Connect and setup etcd interfaces + client, err := GetAndTestEtcdClient(etcdURL) + if err != nil { + return tools.EtcdHelper{}, err + } + + version := latest.Version + interfaces, err := latest.InterfacesFor(version) + if err != nil { + return helper, err + } + return tools.EtcdHelper{client, interfaces.Codec, tools.RuntimeVersionAdapter{interfaces.MetadataAccessor}}, nil +} diff --git a/pkg/cmd/server/kube_master.go b/pkg/cmd/server/kube_master.go deleted file mode 100644 index ced2a62e52a0..000000000000 --- a/pkg/cmd/server/kube_master.go +++ /dev/null @@ -1,71 +0,0 @@ -package server - -import ( - "fmt" - "net" - - kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" - "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" - kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - kmaster "github.com/GoogleCloudPlatform/kubernetes/pkg/master" - - "github.com/openshift/origin/pkg/cmd/server/kubernetes" - - // Admission control plugins from upstream Kubernetes - "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/limitranger" - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/namespace/exists" - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcedefaults" - _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcequota" -) - -func (cfg Config) BuildKubernetesMasterConfig(requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*kubernetes.MasterConfig, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - - // Connect and setup etcd interfaces - etcdClient, err := cfg.getAndTestEtcdClient() - if err != nil { - return nil, err - } - ketcdHelper, err := kmaster.NewEtcdHelper(etcdClient, klatest.Version) - if err != nil { - return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err) - } - - portalNet := net.IPNet(cfg.PortalNet) - masterIP := net.ParseIP(getHost(*masterAddr)) - if masterIP == nil { - addrs, err := net.LookupIP(getHost(*masterAddr)) - if err != nil { - return nil, fmt.Errorf("Unable to find an IP for %q - specify an IP directly? %v", getHost(*masterAddr), err) - } - if len(addrs) == 0 { - return nil, fmt.Errorf("Unable to find an IP for %q - specify an IP directly?", getHost(*masterAddr)) - } - masterIP = addrs[0] - } - - // in-order list of plug-ins that should intercept admission decisions - // TODO: add NamespaceExists - admissionControlPluginNames := []string{"LimitRanger", "ResourceQuota"} - admissionController := admission.NewFromPlugins(kubeClient, admissionControlPluginNames, "") - - kmaster := &kubernetes.MasterConfig{ - MasterIP: masterIP, - MasterPort: cfg.MasterAddr.Port, - NodeHosts: cfg.NodeList, - PortalNet: &portalNet, - RequestContextMapper: requestContextMapper, - EtcdHelper: ketcdHelper, - KubeClient: kubeClient, - Authorizer: apiserver.NewAlwaysAllowAuthorizer(), - AdmissionControl: admissionController, - } - - return kmaster, nil -} diff --git a/pkg/cmd/server/kubernetes/master.go b/pkg/cmd/server/kubernetes/master.go index 170adef3b1c1..98c58e776d4d 100644 --- a/pkg/cmd/server/kubernetes/master.go +++ b/pkg/cmd/server/kubernetes/master.go @@ -2,26 +2,23 @@ package kubernetes import ( "fmt" - "net" "time" - "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" + "github.com/emicklei/go-restful" + "github.com/golang/glog" + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/resource" - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer" kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" minionControllerPkg "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/controller" "github.com/GoogleCloudPlatform/kubernetes/pkg/controller" "github.com/GoogleCloudPlatform/kubernetes/pkg/master" "github.com/GoogleCloudPlatform/kubernetes/pkg/resourcequota" "github.com/GoogleCloudPlatform/kubernetes/pkg/service" - "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" kubeutil "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler" _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithmprovider" "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/factory" - "github.com/emicklei/go-restful" - "github.com/golang/glog" ) const ( @@ -31,22 +28,6 @@ const ( KubeAPIPrefixV1Beta3 = "/api/v1beta3" ) -// MasterConfig defines the required values to start a Kubernetes master -type MasterConfig struct { - MasterIP net.IP - MasterPort int - NodeHosts []string - PortalNet *net.IPNet - - RequestContextMapper kapi.RequestContextMapper - - EtcdHelper tools.EtcdHelper - KubeClient *kclient.Client - - Authorizer authorizer.Authorizer - AdmissionControl admission.Interface -} - // TODO: Longer term we should read this from some config store, rather than a flag. func (c *MasterConfig) EnsurePortalFlags() { if c.PortalNet == nil { diff --git a/pkg/cmd/server/kubernetes/master_config.go b/pkg/cmd/server/kubernetes/master_config.go new file mode 100644 index 000000000000..e577c691eba5 --- /dev/null +++ b/pkg/cmd/server/kubernetes/master_config.go @@ -0,0 +1,83 @@ +package kubernetes + +import ( + "errors" + "fmt" + "net" + "strconv" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + klatest "github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest" + "github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver" + "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/authorizer" + kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/master" + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + + "github.com/openshift/origin/pkg/cmd/flagtypes" + configapi "github.com/openshift/origin/pkg/cmd/server/api" + "github.com/openshift/origin/pkg/cmd/server/etcd" +) + +// MasterConfig defines the required values to start a Kubernetes master +type MasterConfig struct { + MasterIP net.IP + MasterPort int + NodeHosts []string + PortalNet *net.IPNet + + RequestContextMapper kapi.RequestContextMapper + + EtcdHelper tools.EtcdHelper + KubeClient *kclient.Client + + Authorizer authorizer.Authorizer + AdmissionControl admission.Interface +} + +func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*MasterConfig, error) { + if options.KubernetesMasterConfig == nil { + return nil, errors.New("insufficient information to build KubernetesMasterConfig") + } + + // Connect and setup etcd interfaces + etcdClient, err := etcd.GetAndTestEtcdClient(options.EtcdClientInfo.URL) + if err != nil { + return nil, err + } + ketcdHelper, err := master.NewEtcdHelper(etcdClient, klatest.Version) + if err != nil { + return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err) + } + + portalNet := net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet)) + + // in-order list of plug-ins that should intercept admission decisions + // TODO: add NamespaceExists + admissionControlPluginNames := []string{"LimitRanger", "ResourceQuota"} + admissionController := admission.NewFromPlugins(kubeClient, admissionControlPluginNames, "") + + host, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress) + if err != nil { + return nil, err + } + port, err := strconv.Atoi(portString) + if err != nil { + return nil, err + } + + kmaster := &MasterConfig{ + MasterIP: net.ParseIP(host), + MasterPort: port, + NodeHosts: options.KubernetesMasterConfig.StaticNodeNames, + PortalNet: &portalNet, + RequestContextMapper: requestContextMapper, + EtcdHelper: ketcdHelper, + KubeClient: kubeClient, + Authorizer: apiserver.NewAlwaysAllowAuthorizer(), + AdmissionControl: admissionController, + } + + return kmaster, nil +} diff --git a/pkg/cmd/server/kubernetes/node.go b/pkg/cmd/server/kubernetes/node.go index 870e10c2fedf..2267f4a0ff59 100644 --- a/pkg/cmd/server/kubernetes/node.go +++ b/pkg/cmd/server/kubernetes/node.go @@ -2,7 +2,6 @@ package kubernetes import ( "crypto/tls" - "crypto/x509" "fmt" "net" "net/http" @@ -10,11 +9,9 @@ import ( "os/exec" "path/filepath" "reflect" - "strconv" "time" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet" kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config" "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" @@ -55,44 +52,6 @@ func (ce defaultCommandExecutor) Run(command string, args ...string) error { return c.Run() } -// NodeConfig represents the required parameters to start the OpenShift node -// through Kubernetes. All fields are required. -type NodeConfig struct { - // The address to bind to - BindHost string - // The name of this node that will be used to identify the node in the master. - // This value must match the value provided to the master on startup. - NodeHost string - // The host that the master can be reached at (not in use yet) - MasterHost string - // The directory that volumes will be stored under - VolumeDir string - - ClusterDomain string - ClusterDNS net.IP - - // The image used as the Kubelet network namespace and volume container. - NetworkContainerImage string - - // If true, the Kubelet will ignore errors from Docker - AllowDisabledDocker bool - - // Whether to enable TLS serving - TLS bool - - KubeletCertFile string - KubeletKeyFile string - - // ClientCAs will be used to request client certificates in connections to the node. - // This CertPool should contain all the CAs that will be used for client certificate verification. - ClientCAs *x509.CertPool - - // A client to connect to the master. - Client *client.Client - // A client to connect to Docker - DockerClient dockertools.DockerInterface -} - // EnsureDocker attempts to connect to the Docker daemon defined by the helper, // and if it is unable to it will print a warning. func (c *NodeConfig) EnsureDocker(docker *dockerutil.Helper) { @@ -186,7 +145,7 @@ func (c *NodeConfig) RunKubelet() { handler := kubelet.NewServer(k, true) server := &http.Server{ - Addr: net.JoinHostPort(c.BindHost, strconv.Itoa(NodePort)), + Addr: c.BindAddress, Handler: &handler, ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, @@ -194,7 +153,7 @@ func (c *NodeConfig) RunKubelet() { } go util.Forever(func() { - glog.Infof("Started Kubelet for node %s, server at %s:%d", c.NodeHost, c.BindHost, NodePort) + glog.Infof("Started Kubelet for node %s, server at %s", c.NodeHost, c.BindAddress) if clusterDNS != nil { glog.Infof(" Kubelet is setting %s as a DNS nameserver for domain %q", clusterDNS, c.ClusterDomain) } @@ -245,9 +204,13 @@ func (c *NodeConfig) RunProxy() { loadBalancer := proxy.NewLoadBalancerRR() endpointsConfig.RegisterHandler(loadBalancer) - ip := net.ParseIP(c.BindHost) + host, _, err := net.SplitHostPort(c.BindAddress) + if err != nil { + glog.Fatalf("The provided value to bind to must be an ip:port %q", c.BindAddress) + } + ip := net.ParseIP(host) if ip == nil { - glog.Fatalf("The provided value to bind to must be an IP: %q", c.BindHost) + glog.Fatalf("The provided value to bind to must be an ip:port: %q", c.BindAddress) } protocol := iptables.ProtocolIpv4 @@ -263,5 +226,5 @@ func (c *NodeConfig) RunProxy() { } serviceConfig.RegisterHandler(proxier) - glog.Infof("Started Kubernetes Proxy on %s", c.BindHost) + glog.Infof("Started Kubernetes Proxy on %s", host) } diff --git a/pkg/cmd/server/kubernetes/node_config.go b/pkg/cmd/server/kubernetes/node_config.go new file mode 100644 index 000000000000..9836c288802f --- /dev/null +++ b/pkg/cmd/server/kubernetes/node_config.go @@ -0,0 +1,80 @@ +package kubernetes + +import ( + "crypto/x509" + "fmt" + "net" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" +) + +// NodeConfig represents the required parameters to start the OpenShift node +// through Kubernetes. All fields are required. +type NodeConfig struct { + // The address to bind to + BindAddress string + // The name of this node that will be used to identify the node in the master. + // This value must match the value provided to the master on startup. + NodeHost string + // The host that the master can be reached at (not in use yet) + MasterHost string + // The directory that volumes will be stored under + VolumeDir string + + ClusterDomain string + ClusterDNS net.IP + + // The image used as the Kubelet network namespace and volume container. + NetworkContainerImage string + + // If true, the Kubelet will ignore errors from Docker + AllowDisabledDocker bool + + // Whether to enable TLS serving + TLS bool + + KubeletCertFile string + KubeletKeyFile string + + // ClientCAs will be used to request client certificates in connections to the node. + // This CertPool should contain all the CAs that will be used for client certificate verification. + ClientCAs *x509.CertPool + + // A client to connect to the master. + Client *client.Client + // A client to connect to Docker + DockerClient dockertools.DockerInterface +} + +func BuildKubernetesNodeConfig(options configapi.NodeConfig) (*NodeConfig, error) { + kubeClient, _, err := configapi.GetKubeClient(options.MasterKubeConfig) + if err != nil { + return nil, err + } + + var dnsIP net.IP + if len(options.DNSIP) > 0 { + dnsIP = net.ParseIP(options.DNSIP) + if dnsIP == nil { + return nil, fmt.Errorf("Invalid DNS IP: %s", options.DNSIP) + } + } + + config := &NodeConfig{ + NodeHost: options.NodeName, + BindAddress: options.ServingInfo.BindAddress, + + ClusterDomain: options.DNSDomain, + ClusterDNS: dnsIP, + + VolumeDir: options.VolumeDirectory, + NetworkContainerImage: options.NetworkContainerImage, + AllowDisabledDocker: options.AllowDisabledDocker, + Client: kubeClient, + } + + return config, nil +} diff --git a/pkg/cmd/server/kubernetes/proxy.go b/pkg/cmd/server/kubernetes/proxy.go index ce49cf457e34..05c6047f5356 100644 --- a/pkg/cmd/server/kubernetes/proxy.go +++ b/pkg/cmd/server/kubernetes/proxy.go @@ -11,12 +11,16 @@ import ( ) type ProxyConfig struct { - KubernetesAddr *url.URL - ClientConfig *kclient.Config + ClientConfig *kclient.Config } func (c *ProxyConfig) InstallAPI(container *restful.Container) []string { - proxy, err := httpproxy.NewUpgradeAwareSingleHostReverseProxy(c.ClientConfig, c.KubernetesAddr) + kubeAddr, err := url.Parse(c.ClientConfig.Host) + if err != nil { + glog.Fatal(err) + } + + proxy, err := httpproxy.NewUpgradeAwareSingleHostReverseProxy(c.ClientConfig, kubeAddr) if err != nil { glog.Fatalf("Unable to initialize the Kubernetes proxy: %v", err) } diff --git a/pkg/cmd/server/node_config.go b/pkg/cmd/server/node_config.go deleted file mode 100644 index 2ce82bba841b..000000000000 --- a/pkg/cmd/server/node_config.go +++ /dev/null @@ -1,46 +0,0 @@ -package server - -import ( - "net" - - "github.com/openshift/origin/pkg/cmd/server/kubernetes" -) - -func (cfg Config) BuildKubernetesNodeConfig() (*kubernetes.NodeConfig, error) { - kubernetesAddr, err := cfg.GetKubernetesAddress() - if err != nil { - return nil, err - } - kubeClient, _, err := cfg.GetKubeClient() - if err != nil { - return nil, err - } - - dnsDomain := env("OPENSHIFT_DNS_DOMAIN", "local") - dnsIP := cfg.ClusterDNS - if clusterDNS := env("OPENSHIFT_DNS_ADDR", ""); len(clusterDNS) > 0 { - dnsIP = net.ParseIP(clusterDNS) - } - - // define a function for resolving components to names - imageResolverFn := cfg.ImageTemplate.ExpandOrDie - - nodeConfig := &kubernetes.NodeConfig{ - BindHost: cfg.BindAddr.Host, - NodeHost: cfg.Hostname, - MasterHost: kubernetesAddr.String(), - - ClusterDomain: dnsDomain, - ClusterDNS: dnsIP, - - VolumeDir: cfg.VolumeDir, - - NetworkContainerImage: imageResolverFn("pod"), - - AllowDisabledDocker: cfg.StartNode && cfg.StartMaster, - - Client: kubeClient, - } - - return nodeConfig, nil -} diff --git a/pkg/cmd/server/origin/auth.go b/pkg/cmd/server/origin/auth.go index 654e19e026c8..b2faac8ecf6b 100644 --- a/pkg/cmd/server/origin/auth.go +++ b/pkg/cmd/server/origin/auth.go @@ -159,72 +159,6 @@ func ParseAuthRequestHandlerTypes(types string) []AuthRequestHandlerType { return handlerTypes } -type AuthConfig struct { - // URL to call internally during token request - MasterAddr string - // URL to direct browsers to the master on - MasterPublicAddr string - // Valid redirectURI prefixes to direct browsers to the web console - AssetPublicAddresses []string - MasterRoots *x509.CertPool - EtcdHelper tools.EtcdHelper - - // Max age of authorize tokens - AuthorizeTokenMaxAgeSeconds int32 - // Max age of access tokens - AccessTokenMaxAgeSeconds int32 - - // AuthRequestHandlers contains an ordered list of authenticators that decide if a request is authenticated - AuthRequestHandlers []AuthRequestHandlerType - - // AuthHandler specifies what handles unauthenticated requests - AuthHandler AuthHandlerType - - // GrantHandler specifies what handles requests for new client authorizations - GrantHandler GrantHandlerType - - // PasswordAuth specifies how to validate username/passwords. Used by AuthRequestHandlerBasicAuth and AuthHandlerLogin - PasswordAuth PasswordAuthType - // BasicAuthURL specifies the remote URL to validate username/passwords against using basic auth. Used by PasswordAuthBasicAuthURL. - BasicAuthURL string - // HTPasswdFile specifies the path to an htpasswd file to validate username/passwords against. Used by PasswordAuthHTPasswd. - HTPasswdFile string - - // TokenStore specifies how to validate bearer tokens. Used by AuthRequestHandlerBearer. - TokenStore TokenStoreType - // TokenFilePath is a path to a CSV file to load valid tokens from. Used by TokenStoreFile. - TokenFilePath string - - // RequestHeaders lists the headers to check (in order) for a username. Used by AuthRequestHandlerRequestHeader - RequestHeaders []string - // RequestHeaderCAFile specifies the path to a PEM-encoded certificate bundle. - // If set, a client certificate must be presented and validate against the CA before the request headers are checked for usernames - RequestHeaderCAFile string - - // SessionSecrets list the secret(s) to use to encrypt created sessions. Used by AuthRequestHandlerSession - SessionSecrets []string - // SessionMaxAgeSeconds specifies how long created sessions last. Used by AuthRequestHandlerSession - SessionMaxAgeSeconds int32 - // SessionName is the cookie name used to store the session - SessionName string - // sessionAuth holds the Authenticator built from the exported Session* options. It should only be accessed via getSessionAuth(), since it is lazily built. - sessionAuth *session.Authenticator - - // GoogleClientID is the client_id of a client registered with the Google OAuth provider. - // It must be authorized to redirect to {MasterPublicAddr}/oauth2callback/google - // Used by AuthHandlerGoogle - GoogleClientID string - // GoogleClientID is the client_secret of a client registered with the Google OAuth provider. - GoogleClientSecret string - - // GithubClientID is the client_id of a client registered with the GitHub OAuth provider. - // It must be authorized to redirect to {MasterPublicAddr}/oauth2callback/github - // Used by AuthHandlerGithub - GithubClientID string - // GithubClientID is the client_secret of a client registered with the GitHub OAuth provider. - GithubClientSecret string -} - // InstallSupport registers endpoints for an OAuth2 server into the provided mux, // then returns an array of strings indicating what endpoints were started // (these are format strings that will expect to be sent a single string value). diff --git a/pkg/cmd/server/origin/auth_config.go b/pkg/cmd/server/origin/auth_config.go new file mode 100644 index 000000000000..935a8679b141 --- /dev/null +++ b/pkg/cmd/server/origin/auth_config.go @@ -0,0 +1,144 @@ +package origin + +import ( + "crypto/x509" + "fmt" + "strings" + + "code.google.com/p/go-uuid/uuid" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/tools" + + "github.com/openshift/origin/pkg/auth/server/session" + configapi "github.com/openshift/origin/pkg/cmd/server/api" + "github.com/openshift/origin/pkg/cmd/server/etcd" + cmdutil "github.com/openshift/origin/pkg/cmd/util" +) + +type AuthConfig struct { + // URL to call internally during token request + MasterAddr string + // URL to direct browsers to the master on + MasterPublicAddr string + // Valid redirectURI prefixes to direct browsers to the web console + AssetPublicAddresses []string + MasterRoots *x509.CertPool + EtcdHelper tools.EtcdHelper + + // Max age of authorize tokens + AuthorizeTokenMaxAgeSeconds int32 + // Max age of access tokens + AccessTokenMaxAgeSeconds int32 + + // AuthRequestHandlers contains an ordered list of authenticators that decide if a request is authenticated + AuthRequestHandlers []AuthRequestHandlerType + + // AuthHandler specifies what handles unauthenticated requests + AuthHandler AuthHandlerType + + // GrantHandler specifies what handles requests for new client authorizations + GrantHandler GrantHandlerType + + // PasswordAuth specifies how to validate username/passwords. Used by AuthRequestHandlerBasicAuth and AuthHandlerLogin + PasswordAuth PasswordAuthType + // BasicAuthURL specifies the remote URL to validate username/passwords against using basic auth. Used by PasswordAuthBasicAuthURL. + BasicAuthURL string + // HTPasswdFile specifies the path to an htpasswd file to validate username/passwords against. Used by PasswordAuthHTPasswd. + HTPasswdFile string + + // TokenStore specifies how to validate bearer tokens. Used by AuthRequestHandlerBearer. + TokenStore TokenStoreType + // TokenFilePath is a path to a CSV file to load valid tokens from. Used by TokenStoreFile. + TokenFilePath string + + // RequestHeaders lists the headers to check (in order) for a username. Used by AuthRequestHandlerRequestHeader + RequestHeaders []string + // RequestHeaderCAFile specifies the path to a PEM-encoded certificate bundle. + // If set, a client certificate must be presented and validate against the CA before the request headers are checked for usernames + RequestHeaderCAFile string + + // SessionSecrets list the secret(s) to use to encrypt created sessions. Used by AuthRequestHandlerSession + SessionSecrets []string + // SessionMaxAgeSeconds specifies how long created sessions last. Used by AuthRequestHandlerSession + SessionMaxAgeSeconds int32 + // SessionName is the cookie name used to store the session + SessionName string + // sessionAuth holds the Authenticator built from the exported Session* options. It should only be accessed via getSessionAuth(), since it is lazily built. + sessionAuth *session.Authenticator + + // GoogleClientID is the client_id of a client registered with the Google OAuth provider. + // It must be authorized to redirect to {MasterPublicAddr}/oauth2callback/google + // Used by AuthHandlerGoogle + GoogleClientID string + // GoogleClientID is the client_secret of a client registered with the Google OAuth provider. + GoogleClientSecret string + + // GithubClientID is the client_id of a client registered with the GitHub OAuth provider. + // It must be authorized to redirect to {MasterPublicAddr}/oauth2callback/github + // Used by AuthHandlerGithub + GithubClientID string + // GithubClientID is the client_secret of a client registered with the GitHub OAuth provider. + GithubClientSecret string +} + +func BuildAuthConfig(options configapi.MasterConfig) (*AuthConfig, error) { + etcdHelper, err := etcd.NewOpenShiftEtcdHelper(options.EtcdClientInfo.URL) + if err != nil { + return nil, fmt.Errorf("Error setting up server storage: %v", err) + } + + apiServerCAs, err := configapi.GetAPIServerCertCAPool(options) + if err != nil { + return nil, err + } + + // Build the list of valid redirect_uri prefixes for a login using the openshift-web-console client to redirect to + // TODO: allow configuring this + // TODO: remove hard-coding of development UI server + assetPublicURLs := []string{options.OAuthConfig.AssetPublicURL, "http://localhost:9000", "https://localhost:9000"} + + // Default to a session authenticator (for browsers), and a basicauth authenticator (for clients responding to WWW-Authenticate challenges) + defaultAuthRequestHandlers := strings.Join([]string{ + string(AuthRequestHandlerSession), + string(AuthRequestHandlerBasicAuth), + }, ",") + + ret := &AuthConfig{ + MasterAddr: options.OAuthConfig.MasterURL, + MasterPublicAddr: options.OAuthConfig.MasterPublicURL, + AssetPublicAddresses: assetPublicURLs, + MasterRoots: apiServerCAs, + EtcdHelper: etcdHelper, + + // Max token ages + AuthorizeTokenMaxAgeSeconds: cmdutil.EnvInt("OPENSHIFT_OAUTH_AUTHORIZE_TOKEN_MAX_AGE_SECONDS", 300, 1), + AccessTokenMaxAgeSeconds: cmdutil.EnvInt("OPENSHIFT_OAUTH_ACCESS_TOKEN_MAX_AGE_SECONDS", 3600, 1), + // Handlers + AuthRequestHandlers: ParseAuthRequestHandlerTypes(cmdutil.Env("OPENSHIFT_OAUTH_REQUEST_HANDLERS", defaultAuthRequestHandlers)), + AuthHandler: AuthHandlerType(cmdutil.Env("OPENSHIFT_OAUTH_HANDLER", string(AuthHandlerLogin))), + GrantHandler: GrantHandlerType(cmdutil.Env("OPENSHIFT_OAUTH_GRANT_HANDLER", string(GrantHandlerAuto))), + // RequestHeader config + RequestHeaders: strings.Split(cmdutil.Env("OPENSHIFT_OAUTH_REQUEST_HEADERS", "X-Remote-User"), ","), + RequestHeaderCAFile: options.OAuthConfig.ProxyCA, + // Session config (default to unknowable secret) + SessionSecrets: []string{cmdutil.Env("OPENSHIFT_OAUTH_SESSION_SECRET", uuid.NewUUID().String())}, + SessionMaxAgeSeconds: cmdutil.EnvInt("OPENSHIFT_OAUTH_SESSION_MAX_AGE_SECONDS", 300, 1), + SessionName: cmdutil.Env("OPENSHIFT_OAUTH_SESSION_NAME", "ssn"), + // Password config + PasswordAuth: PasswordAuthType(cmdutil.Env("OPENSHIFT_OAUTH_PASSWORD_AUTH", string(PasswordAuthAnyPassword))), + BasicAuthURL: cmdutil.Env("OPENSHIFT_OAUTH_BASIC_AUTH_URL", ""), + HTPasswdFile: cmdutil.Env("OPENSHIFT_OAUTH_HTPASSWD_FILE", ""), + // Token config + TokenStore: TokenStoreType(cmdutil.Env("OPENSHIFT_OAUTH_TOKEN_STORE", string(TokenStoreOAuth))), + TokenFilePath: cmdutil.Env("OPENSHIFT_OAUTH_TOKEN_FILE_PATH", ""), + // Google config + GoogleClientID: cmdutil.Env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_ID", ""), + GoogleClientSecret: cmdutil.Env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_SECRET", ""), + // GitHub config + GithubClientID: cmdutil.Env("OPENSHIFT_OAUTH_GITHUB_CLIENT_ID", ""), + GithubClientSecret: cmdutil.Env("OPENSHIFT_OAUTH_GITHUB_CLIENT_SECRET", ""), + } + + return ret, nil + +} diff --git a/pkg/cmd/server/origin/master.go b/pkg/cmd/server/origin/master.go index c22df2c36bdf..dce809a82e1b 100644 --- a/pkg/cmd/server/origin/master.go +++ b/pkg/cmd/server/origin/master.go @@ -86,6 +86,7 @@ import ( roleregistry "github.com/openshift/origin/pkg/authorization/registry/role" rolebindingregistry "github.com/openshift/origin/pkg/authorization/registry/rolebinding" subjectaccessreviewregistry "github.com/openshift/origin/pkg/authorization/registry/subjectaccessreview" + configapi "github.com/openshift/origin/pkg/cmd/server/api" "github.com/openshift/origin/pkg/cmd/server/bootstrappolicy" routeplugin "github.com/openshift/origin/plugins/route/allocation/simple" ) @@ -114,7 +115,7 @@ func (fn APIInstallFunc) InstallAPI(container *restful.Container) []string { // KubeClient returns the kubernetes client object func (c *MasterConfig) KubeClient() *kclient.Client { - return c.MasterConfigParameters.KubeClient + return c.KubernetesClient } // PolicyClient returns the policy client object @@ -128,19 +129,19 @@ func (c *MasterConfig) PolicyClient() *osclient.Client { // DeploymentClient returns the deployment client object func (c *MasterConfig) DeploymentClient() *kclient.Client { - return c.MasterConfigParameters.KubeClient + return c.KubernetesClient } // DNSServerClient returns the DNS server client object // It must have the following capabilities: // list, watch all services in all namespaces func (c *MasterConfig) DNSServerClient() *kclient.Client { - return c.MasterConfigParameters.KubeClient + return c.KubernetesClient } // BuildLogClient returns the build log client object func (c *MasterConfig) BuildLogClient() *kclient.Client { - return c.MasterConfigParameters.KubeClient + return c.KubernetesClient } // WebHookClient returns the webhook client object @@ -150,7 +151,7 @@ func (c *MasterConfig) WebHookClient() *osclient.Client { // BuildControllerClients returns the build controller client objects func (c *MasterConfig) BuildControllerClients() (*osclient.Client, *kclient.Client) { - return c.OSClient, c.MasterConfigParameters.KubeClient + return c.OSClient, c.KubernetesClient } // ImageChangeControllerClient returns the openshift client object @@ -160,7 +161,7 @@ func (c *MasterConfig) ImageChangeControllerClient() *osclient.Client { // DeploymentControllerClients returns the deployment controller client object func (c *MasterConfig) DeploymentControllerClients() (*osclient.Client, *kclient.Client) { - return c.OSClient, c.MasterConfigParameters.KubeClient + return c.OSClient, c.KubernetesClient } // DeployerClientConfig returns the client configuration a Deployer instance launched in a pod @@ -170,10 +171,10 @@ func (c *MasterConfig) DeployerClientConfig() *kclient.Config { } func (c *MasterConfig) DeploymentConfigControllerClients() (*osclient.Client, *kclient.Client) { - return c.OSClient, c.MasterConfigParameters.KubeClient + return c.OSClient, c.KubernetesClient } func (c *MasterConfig) DeploymentConfigChangeControllerClients() (*osclient.Client, *kclient.Client) { - return c.OSClient, c.MasterConfigParameters.KubeClient + return c.OSClient, c.KubernetesClient } func (c *MasterConfig) DeploymentImageChangeControllerClient() *osclient.Client { return c.OSClient @@ -258,7 +259,7 @@ func (c *MasterConfig) InstallProtectedAPI(container *restful.Container) []strin "policies": policyregistry.NewREST(authorizationEtcd), "policyBindings": policybindingregistry.NewREST(authorizationEtcd), "roles": roleregistry.NewREST(roleregistry.NewVirtualRegistry(authorizationEtcd)), - "roleBindings": rolebindingregistry.NewREST(rolebindingregistry.NewVirtualRegistry(authorizationEtcd, authorizationEtcd, c.MasterAuthorizationNamespace)), + "roleBindings": rolebindingregistry.NewREST(rolebindingregistry.NewVirtualRegistry(authorizationEtcd, authorizationEtcd, c.Options.MasterAuthorizationNamespace)), "resourceAccessReviews": resourceaccessreviewregistry.NewREST(c.Authorizer), "subjectAccessReviews": subjectaccessreviewregistry.NewREST(c.Authorizer), } @@ -365,7 +366,7 @@ func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) } server := &http.Server{ - Addr: c.MasterBindAddr, + Addr: c.Options.ServingInfo.BindAddress, Handler: handler, ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, @@ -374,7 +375,7 @@ func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) go util.Forever(func() { for _, s := range extra { - glog.Infof(s, c.MasterAddr) + glog.Infof(s, c.Options.ServingInfo.BindAddress) } if c.TLS { server.TLSConfig = &tls.Config{ @@ -385,14 +386,14 @@ func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) ClientAuth: tls.RequestClientCert, ClientCAs: c.ClientCAs, } - glog.Fatal(server.ListenAndServeTLS(c.MasterCertFile, c.MasterKeyFile)) + glog.Fatal(server.ListenAndServeTLS(c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile)) } else { glog.Fatal(server.ListenAndServe()) } }, 0) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) - cmdutil.WaitForSuccessfulDial(c.TLS, "tcp", c.MasterBindAddr, 100*time.Millisecond, 100*time.Millisecond, 100) + cmdutil.WaitForSuccessfulDial(c.TLS, "tcp", c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) // Attempt to create the required policy rules now, and then stick in a forever loop to make sure they are always available c.ensureComponentAuthorizationRules() @@ -415,9 +416,9 @@ func (c *MasterConfig) getRequestContextMapper() kapi.RequestContextMapper { func (c *MasterConfig) ensureMasterAuthorizationNamespace() { // ensure that master namespace actually exists - namespace, err := c.KubeClient().Namespaces().Get(c.MasterAuthorizationNamespace) + namespace, err := c.KubeClient().Namespaces().Get(c.Options.MasterAuthorizationNamespace) if err != nil { - namespace = &kapi.Namespace{ObjectMeta: kapi.ObjectMeta{Name: c.MasterAuthorizationNamespace}} + namespace = &kapi.Namespace{ObjectMeta: kapi.ObjectMeta{Name: c.Options.MasterAuthorizationNamespace}} kapi.FillObjectMetaSystemFields(api.NewContext(), &namespace.ObjectMeta) _, err = c.KubeClient().Namespaces().Create(namespace) if err != nil { @@ -431,7 +432,7 @@ func (c *MasterConfig) ensureComponentAuthorizationRules() { registry := authorizationetcd.New(c.EtcdHelper) roleRegistry := roleregistry.NewVirtualRegistry(registry) - for _, role := range bootstrappolicy.GetBootstrapRoles(c.MasterAuthorizationNamespace, c.OpenshiftSharedResourcesNamespace) { + for _, role := range bootstrappolicy.GetBootstrapRoles(c.Options.MasterAuthorizationNamespace, c.Options.OpenShiftSharedResourcesNamespace) { ctx := kapi.WithNamespace(kapi.NewContext(), role.Namespace) if _, err := roleRegistry.GetRole(ctx, role.Name); kapierror.IsNotFound(err) { @@ -444,13 +445,13 @@ func (c *MasterConfig) ensureComponentAuthorizationRules() { } } - roleBindingRegistry := rolebindingregistry.NewVirtualRegistry(registry, registry, c.MasterAuthorizationNamespace) - for _, roleBinding := range bootstrappolicy.GetBootstrapRoleBindings(c.MasterAuthorizationNamespace, c.OpenshiftSharedResourcesNamespace) { + roleBindingRegistry := rolebindingregistry.NewVirtualRegistry(registry, registry, c.Options.MasterAuthorizationNamespace) + for _, roleBinding := range bootstrappolicy.GetBootstrapRoleBindings(c.Options.MasterAuthorizationNamespace, c.Options.OpenShiftSharedResourcesNamespace) { ctx := kapi.WithNamespace(kapi.NewContext(), roleBinding.Namespace) if _, err := roleBindingRegistry.GetRoleBinding(ctx, roleBinding.Name); kapierror.IsNotFound(err) { // if this is a binding for a non-master namespaced role. That means that the policy binding must be provisioned - if roleBinding.RoleRef.Namespace != c.MasterAuthorizationNamespace { + if roleBinding.RoleRef.Namespace != c.Options.MasterAuthorizationNamespace { policyBindingName := roleBinding.RoleRef.Namespace if _, err := registry.GetPolicyBinding(ctx, policyBindingName); kapierror.IsNotFound(err) { policyBinding := &authorizationapi.PolicyBinding{ @@ -540,12 +541,12 @@ func (c *MasterConfig) RunAssetServer() { // TODO use version.Get().GitCommit as an etag cache header mux := http.NewServeMux() - masterURL, err := url.Parse(c.MasterPublicAddr) + masterURL, err := url.Parse(c.Options.AssetConfig.MasterPublicURL) if err != nil { glog.Fatalf("Error parsing master url: %v", err) } - k8sURL, err := url.Parse(c.KubernetesPublicAddr) + k8sURL, err := url.Parse(c.Options.AssetConfig.KubernetesPublicURL) if err != nil { glog.Fatalf("Error parsing kubernetes url: %v", err) } @@ -556,9 +557,9 @@ func (c *MasterConfig) RunAssetServer() { KubernetesAddr: k8sURL.Host, KubernetesPrefix: "/api", OAuthAuthorizeURI: OpenShiftOAuthAuthorizeURL(masterURL.String()), - OAuthRedirectBase: c.AssetPublicAddr, + OAuthRedirectBase: c.Options.AssetConfig.PublicURL, OAuthClientID: OpenShiftWebConsoleClientID, - LogoutURI: c.LogoutURI, + LogoutURI: c.Options.AssetConfig.LogoutURI, } assets.RegisterMimeTypes() @@ -588,7 +589,7 @@ func (c *MasterConfig) RunAssetServer() { ) server := &http.Server{ - Addr: c.AssetBindAddr, + Addr: c.Options.AssetConfig.ServingInfo.BindAddress, Handler: mux, ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, @@ -601,18 +602,18 @@ func (c *MasterConfig) RunAssetServer() { // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) MinVersion: tls.VersionTLS10, } - glog.Infof("OpenShift UI listening at https://%s", c.AssetBindAddr) - glog.Fatal(server.ListenAndServeTLS(c.AssetCertFile, c.AssetKeyFile)) + glog.Infof("OpenShift UI listening at https://%s", c.Options.AssetConfig.ServingInfo.BindAddress) + glog.Fatal(server.ListenAndServeTLS(c.Options.AssetConfig.ServingInfo.ServerCert.CertFile, c.Options.AssetConfig.ServingInfo.ServerCert.KeyFile)) } else { - glog.Infof("OpenShift UI listening at https://%s", c.AssetBindAddr) + glog.Infof("OpenShift UI listening at https://%s", c.Options.AssetConfig.ServingInfo.BindAddress) glog.Fatal(server.ListenAndServe()) } }, 0) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) - cmdutil.WaitForSuccessfulDial(c.TLS, "tcp", c.AssetBindAddr, 100*time.Millisecond, 100*time.Millisecond, 100) + cmdutil.WaitForSuccessfulDial(c.TLS, "tcp", c.Options.AssetConfig.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) - glog.Infof("OpenShift UI available at %s", c.AssetPublicAddr) + glog.Infof("OpenShift UI available at %s", c.Options.AssetConfig.PublicURL) } func (c *MasterConfig) RunDNSServer() { @@ -620,22 +621,29 @@ func (c *MasterConfig) RunDNSServer() { if err != nil { glog.Fatalf("Could not start DNS: %v", err) } + config.DnsAddr = c.Options.DNSConfig.BindAddress - if _, port, err := net.SplitHostPort(c.DNSBindAddr); err == nil { - if len(port) != 0 && port != "53" { - glog.Warningf("Unable to bind DNS on port 53 (you may need to run as root), using %s which will not resolve from all locations", c.DNSBindAddr) - } + _, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress) + if err != nil { + glog.Fatalf("Could not start DNS: %v", err) + } + if port != "53" { + glog.Warningf("Binding DNS on port %v instead of 53 (you may need to run as root and update your config), using %s which will not resolve from all locations", port, c.Options.DNSConfig.BindAddress) + } + + if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindAddress); !ok { + glog.Warningf("Could not start DNS: %v", err) + return } - config.DnsAddr = c.DNSBindAddr go func() { err := dns.ListenAndServe(config, c.DNSServerClient(), c.EtcdHelper.Client.(*etcdclient.Client)) glog.Fatalf("Could not start DNS: %v", err) }() - cmdutil.WaitForSuccessfulDial(false, "tcp", c.DNSBindAddr, 100*time.Millisecond, 100*time.Millisecond, 100) + cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) - glog.Infof("OpenShift DNS listening at %s", c.DNSBindAddr) + glog.Infof("OpenShift DNS listening at %s", c.Options.DNSConfig.BindAddress) } // RunBuildController starts the build sync loop for builds and buildConfig processing. @@ -682,7 +690,7 @@ func (c *MasterConfig) RunBuildPodController() { controller.Run() } -// RunDeploymentController starts the build image change trigger controller process. +// RunBuildImageChangeTriggerController starts the build image change trigger controller process. func (c *MasterConfig) RunBuildImageChangeTriggerController() { bcClient, _ := c.BuildControllerClients() bcUpdater := buildclient.NewOSClientBuildConfigClient(bcClient) @@ -692,11 +700,17 @@ func (c *MasterConfig) RunBuildImageChangeTriggerController() { } // RunDeploymentController starts the deployment controller process. -func (c *MasterConfig) RunDeploymentController() { +func (c *MasterConfig) RunDeploymentController() error { _, kclient := c.DeploymentControllerClients() + + _, kclientConfig, err := configapi.GetKubeClient(c.Options.MasterClients.OpenShiftLoopbackKubeConfig) + if err != nil { + return err + } + // TODO eliminate these environment variables once we figure out what they do env := []api.EnvVar{ - {Name: "KUBERNETES_MASTER", Value: c.MasterAddr}, - {Name: "OPENSHIFT_MASTER", Value: c.MasterAddr}, + {Name: "KUBERNETES_MASTER", Value: kclientConfig.Host}, + {Name: "OPENSHIFT_MASTER", Value: kclientConfig.Host}, } env = append(env, clientcmd.EnvVarsFromConfig(c.DeployerClientConfig())...) @@ -709,6 +723,8 @@ func (c *MasterConfig) RunDeploymentController() { controller := factory.Create() controller.Run() + + return nil } // RunDeployerPodController starts the deployer pod controller process. @@ -774,10 +790,10 @@ func (c *MasterConfig) RouteAllocator() *routeallocationcontroller.RouteAllocati // ensureCORSAllowedOrigins takes a string list of origins and attempts to covert them to CORS origin // regexes, or exits if it cannot. func (c *MasterConfig) ensureCORSAllowedOrigins() []*regexp.Regexp { - if len(c.CORSAllowedOrigins) == 0 { + if len(c.Options.CORSAllowedOrigins) == 0 { return []*regexp.Regexp{} } - allowedOriginRegexps, err := util.CompileRegexps(util.StringList(c.CORSAllowedOrigins)) + allowedOriginRegexps, err := util.CompileRegexps(util.StringList(c.Options.CORSAllowedOrigins)) if err != nil { glog.Fatalf("Invalid --cors-allowed-origins: %v", err) } diff --git a/pkg/cmd/server/origin/config.go b/pkg/cmd/server/origin/master_config.go similarity index 68% rename from pkg/cmd/server/origin/config.go rename to pkg/cmd/server/origin/master_config.go index d4c4ebe2e532..de8b86fe8913 100644 --- a/pkg/cmd/server/origin/config.go +++ b/pkg/cmd/server/origin/master_config.go @@ -2,8 +2,8 @@ package origin import ( "crypto/x509" + "fmt" "net/http" - "strings" "github.com/GoogleCloudPlatform/kubernetes/pkg/admission" kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" @@ -26,8 +26,12 @@ import ( authorizationetcd "github.com/openshift/origin/pkg/authorization/registry/etcd" "github.com/openshift/origin/pkg/authorization/rulevalidation" osclient "github.com/openshift/origin/pkg/client" + configapi "github.com/openshift/origin/pkg/cmd/server/api" oauthetcd "github.com/openshift/origin/pkg/oauth/registry/etcd" projectauth "github.com/openshift/origin/pkg/project/auth" + + "github.com/openshift/origin/pkg/cmd/server/etcd" + "github.com/openshift/origin/pkg/cmd/util/variable" ) const ( @@ -37,119 +41,128 @@ const ( unauthenticatedGroup = "system:unauthenticated" ) -type MasterConfigParameters struct { - // host:port to bind master to - MasterBindAddr string - // host:port to bind asset server to - AssetBindAddr string - // url to access the master API on within the cluster - MasterAddr string - // url to access kubernetes API on within the cluster - KubernetesAddr string - // external clients may need to access APIs at different addresses than internal components do - MasterPublicAddr string - KubernetesPublicAddr string - AssetPublicAddr string - // host:port to bind DNS on. The default is port 53. - DNSBindAddr string - // LogoutURI is an optional, absolute URI to redirect web browsers to after logging out of the web console. - // If not specified, the built-in logout page is shown. - LogoutURI string - - CORSAllowedOrigins []string +// MasterConfig defines the required parameters for starting the OpenShift master +type MasterConfig struct { + Options configapi.MasterConfig - EtcdHelper tools.EtcdHelper + Authenticator authenticator.Request + Authorizer authorizer.Authorizer + AuthorizationAttributeBuilder authorizer.AuthorizationAttributeBuilder + + PolicyCache *policycache.PolicyCache + ProjectAuthorizationCache *projectauth.AuthorizationCache - MasterCertFile string - MasterKeyFile string - AssetCertFile string - AssetKeyFile string + // Map requests to contexts + RequestContextMapper kapi.RequestContextMapper - // ClientCAs will be used to request client certificates in connections to the API or OAuth server. - // This CertPool should contain all the CAs that will be used for client certificate verification (the union - // of APIClientCAs and OAuthClientCAs). - ClientCAs *x509.CertPool - // APIClientCAs is used to verify client certificates presented for API auth - APIClientCAs *x509.CertPool + AdmissionControl admission.Interface - MasterAuthorizationNamespace string - OpenshiftSharedResourcesNamespace string + TLS bool // a function that returns the appropriate image to use for a named component ImageFor func(component string) string - // kubeClient is the client used to call Kubernetes APIs from system components, built from KubeClientConfig. - // It should only be accessed via the *Client() helper methods. - // To apply different access control to a system component, create a separate client/config specifically for that component. - KubeClient *kclient.Client + EtcdHelper tools.EtcdHelper + + // ClientCAs will be used to request client certificates in connections to the API. + // This CertPool should contain all the CAs that will be used for client certificate verification. + ClientCAs *x509.CertPool + // APIClientCAs is used to verify client certificates presented for API auth + APIClientCAs *x509.CertPool + // KubeClientConfig is the client configuration used to call Kubernetes APIs from system components. // To apply different access control to a system component, create a client config specifically for that component. KubeClientConfig kclient.Config - - // osClient is the client used to call OpenShift APIs from system components, built from OSClientConfig. - // It should only be accessed via the *Client() helper methods. - // To apply different access control to a system component, create a separate client/config specifically for that component. - OSClient *osclient.Client // OSClientConfig is the client configuration used to call OpenShift APIs from system components // To apply different access control to a system component, create a client config specifically for that component. OSClientConfig kclient.Config - // DeployerOSClientConfig is the client configuration used to call OpenShift APIs from launched deployer pods DeployerOSClientConfig kclient.Config -} - -// MasterConfig defines the required parameters for starting the OpenShift master -type MasterConfig struct { - MasterConfigParameters - Authenticator authenticator.Request - Authorizer authorizer.Authorizer - AuthorizationAttributeBuilder authorizer.AuthorizationAttributeBuilder + // kubeClient is the client used to call Kubernetes APIs from system components, built from KubeClientConfig. + // It should only be accessed via the *Client() helper methods. + // To apply different access control to a system component, create a separate client/config specifically for that component. + KubernetesClient *kclient.Client + // osClient is the client used to call OpenShift APIs from system components, built from OSClientConfig. + // It should only be accessed via the *Client() helper methods. + // To apply different access control to a system component, create a separate client/config specifically for that component. + OSClient *osclient.Client +} - PolicyCache *policycache.PolicyCache - ProjectAuthorizationCache *projectauth.AuthorizationCache +func BuildMasterConfig(options configapi.MasterConfig) (*MasterConfig, error) { - // Map requests to contexts - RequestContextMapper kapi.RequestContextMapper + etcdHelper, err := etcd.NewOpenShiftEtcdHelper(options.EtcdClientInfo.URL) + if err != nil { + return nil, fmt.Errorf("Error setting up server storage: %v", err) + } - AdmissionControl admission.Interface + clientCAs, err := configapi.GetClientCertCAPool(options) + if err != nil { + return nil, err + } + apiClientCAs, err := configapi.GetAPIClientCertCAPool(options) + if err != nil { + return nil, err + } - TLS bool -} + kubeClient, kubeClientConfig, err := configapi.GetKubeClient(options.MasterClients.KubernetesKubeConfig) + if err != nil { + return nil, err + } + openshiftClient, osClientConfig, err := configapi.GetOpenShiftClient(options.MasterClients.OpenShiftLoopbackKubeConfig) + if err != nil { + return nil, err + } + _, deployerOSClientConfig, err := configapi.GetOpenShiftClient(options.MasterClients.DeployerKubeConfig) + if err != nil { + return nil, err + } -func BuildMasterConfig(configParams MasterConfigParameters) (*MasterConfig, error) { + imageTemplate := variable.NewDefaultImageTemplate() + imageTemplate.Format = options.ImageConfig.Format + imageTemplate.Latest = options.ImageConfig.Latest - policyCache := configParams.newPolicyCache() + policyCache := newPolicyCache(etcdHelper) requestContextMapper := kapi.NewRequestContextMapper() // in-order list of plug-ins that should intercept admission decisions (origin only intercepts) admissionControlPluginNames := []string{"AlwaysAdmit"} - admissionController := admission.NewFromPlugins(configParams.KubeClient, admissionControlPluginNames, "") + admissionController := admission.NewFromPlugins(kubeClient, admissionControlPluginNames, "") config := &MasterConfig{ - MasterConfigParameters: configParams, + Options: options, - Authenticator: configParams.newAuthenticator(), - Authorizer: newAuthorizer(policyCache, configParams.MasterAuthorizationNamespace), + Authenticator: newAuthenticator(options.ServingInfo, etcdHelper, apiClientCAs), + Authorizer: newAuthorizer(policyCache, options.MasterAuthorizationNamespace), AuthorizationAttributeBuilder: newAuthorizationAttributeBuilder(requestContextMapper), PolicyCache: policyCache, - ProjectAuthorizationCache: configParams.newProjectAuthorizationCache(), + ProjectAuthorizationCache: newProjectAuthorizationCache(options.MasterAuthorizationNamespace, openshiftClient, kubeClient), RequestContextMapper: requestContextMapper, AdmissionControl: admissionController, - TLS: strings.HasPrefix(configParams.MasterAddr, "https://"), + TLS: configapi.UseTLS(options.ServingInfo), + + ImageFor: imageTemplate.ExpandOrDie, + EtcdHelper: etcdHelper, + + ClientCAs: clientCAs, + APIClientCAs: apiClientCAs, + + KubeClientConfig: *kubeClientConfig, + OSClientConfig: *osClientConfig, + DeployerOSClientConfig: *deployerOSClientConfig, + OSClient: openshiftClient, + KubernetesClient: kubeClient, } return config, nil } -func (c MasterConfigParameters) newAuthenticator() authenticator.Request { - useTLS := strings.HasPrefix(c.MasterAddr, "https://") - - tokenAuthenticator := getEtcdTokenAuthenticator(c.EtcdHelper) +func newAuthenticator(servingInfo configapi.ServingInfo, etcdHelper tools.EtcdHelper, apiClientCAs *x509.CertPool) authenticator.Request { + tokenAuthenticator := getEtcdTokenAuthenticator(etcdHelper) authenticators := []authenticator.Request{} authenticators = append(authenticators, bearertoken.New(tokenAuthenticator, true)) @@ -159,11 +172,11 @@ func (c MasterConfigParameters) newAuthenticator() authenticator.Request { // TODO: prevent access_token param from getting logged, if possible authenticators = append(authenticators, paramtoken.New("access_token", tokenAuthenticator, true)) - if useTLS { + if configapi.UseTLS(servingInfo) { // build cert authenticator // TODO: add cert users to etcd? opts := x509request.DefaultVerifyOptions() - opts.Roots = c.APIClientCAs + opts.Roots = apiClientCAs certauth := x509request.New(opts, x509request.SubjectToUserConversion) authenticators = append(authenticators, certauth) } @@ -182,17 +195,17 @@ func (c MasterConfigParameters) newAuthenticator() authenticator.Request { return ret } -func (c MasterConfigParameters) newProjectAuthorizationCache() *projectauth.AuthorizationCache { +func newProjectAuthorizationCache(masterAuthorizationNamespace string, openshiftClient *osclient.Client, kubeClient *kclient.Client) *projectauth.AuthorizationCache { return projectauth.NewAuthorizationCache( - projectauth.NewReviewer(c.OSClient), - c.KubeClient.Namespaces(), - c.OSClient, - c.OSClient, - c.MasterAuthorizationNamespace) + projectauth.NewReviewer(openshiftClient), + kubeClient.Namespaces(), + openshiftClient, + openshiftClient, + masterAuthorizationNamespace) } -func (c MasterConfigParameters) newPolicyCache() *policycache.PolicyCache { - authorizationEtcd := authorizationetcd.New(c.EtcdHelper) +func newPolicyCache(etcdHelper tools.EtcdHelper) *policycache.PolicyCache { + authorizationEtcd := authorizationetcd.New(etcdHelper) return policycache.NewPolicyCache(authorizationEtcd, authorizationEtcd) } diff --git a/pkg/cmd/server/origin_master.go b/pkg/cmd/server/origin_master.go deleted file mode 100644 index c9fc55cd92b4..000000000000 --- a/pkg/cmd/server/origin_master.go +++ /dev/null @@ -1,506 +0,0 @@ -package server - -import ( - "crypto/x509" - "fmt" - "io/ioutil" - "net" - "strings" - "time" - - "code.google.com/p/go-uuid/uuid" - - "github.com/GoogleCloudPlatform/kubernetes/pkg/auth/user" - kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" - clientcmdapi "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd/api" - "github.com/GoogleCloudPlatform/kubernetes/pkg/util" - - osclient "github.com/openshift/origin/pkg/client" - "github.com/openshift/origin/pkg/cmd/server/crypto" - "github.com/openshift/origin/pkg/cmd/server/origin" - cmdutil "github.com/openshift/origin/pkg/cmd/util" -) - -func (cfg Config) BuildOriginMasterConfig() (*origin.MasterConfig, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - kubeAddr, err := cfg.GetKubernetesAddress() - if err != nil { - return nil, err - } - masterPublicAddr, err := cfg.GetMasterPublicAddress() - if err != nil { - return nil, err - } - kubePublicAddr, err := cfg.GetKubernetesPublicAddress() - if err != nil { - return nil, err - } - assetPublicAddr, err := cfg.GetAssetPublicAddress() - if err != nil { - return nil, err - } - - corsAllowedOrigins := []string{} - corsAllowedOrigins = append(corsAllowedOrigins, cfg.CORSAllowedOrigins...) - // always include the all-in-one server's web console as an allowed CORS origin - // always include localhost as an allowed CORS origin - // always include master public address as an allowed CORS origin - for _, origin := range []string{assetPublicAddr.Host, masterPublicAddr.Host, "localhost", "127.0.0.1"} { - // TODO: check if origin is already allowed - corsAllowedOrigins = append(corsAllowedOrigins, origin) - } - - etcdHelper, err := cfg.newOpenShiftEtcdHelper() - if err != nil { - return nil, fmt.Errorf("Error setting up server storage: %v", err) - } - - masterCertFile, masterKeyFile, err := cfg.GetMasterCert() - if err != nil { - return nil, err - } - assetCertFile, assetKeyFile, err := cfg.GetAssetCert() - if err != nil { - return nil, err - } - - clientCAs, err := cfg.GetClientCertCAPool() - if err != nil { - return nil, err - } - apiClientCAs, err := cfg.GetAPIClientCertCAPool() - if err != nil { - return nil, err - } - - kubeClient, kubeClientConfig, err := cfg.GetKubeClient() - if err != nil { - return nil, err - } - openshiftClient, openshiftClientConfig, err := cfg.GetOpenshiftClient() - if err != nil { - return nil, err - } - deployerClientConfig, err := cfg.GetOpenshiftDeployerClientConfig() - if err != nil { - return nil, err - } - - dnsAddr := cfg.DNSBindAddr - if !cmdutil.TryListen(dnsAddr.URL.Host) { - dnsAddr.DefaultPort = 8053 - dnsAddr = dnsAddr.Default() - } - - openshiftConfigParameters := origin.MasterConfigParameters{ - MasterBindAddr: cfg.BindAddr.URL.Host, - AssetBindAddr: cfg.GetAssetBindAddress(), - DNSBindAddr: dnsAddr.URL.Host, - MasterAddr: masterAddr.String(), - KubernetesAddr: kubeAddr.String(), - MasterPublicAddr: masterPublicAddr.String(), - KubernetesPublicAddr: kubePublicAddr.String(), - AssetPublicAddr: assetPublicAddr.String(), - - CORSAllowedOrigins: corsAllowedOrigins, - MasterAuthorizationNamespace: "master", - OpenshiftSharedResourcesNamespace: "openshift", - LogoutURI: env("OPENSHIFT_LOGOUT_URI", ""), - - EtcdHelper: etcdHelper, - - MasterCertFile: masterCertFile, - MasterKeyFile: masterKeyFile, - AssetCertFile: assetCertFile, - AssetKeyFile: assetKeyFile, - ClientCAs: clientCAs, - APIClientCAs: apiClientCAs, - - KubeClient: kubeClient, - KubeClientConfig: *kubeClientConfig, - OSClient: openshiftClient, - OSClientConfig: *openshiftClientConfig, - DeployerOSClientConfig: *deployerClientConfig, - - ImageFor: cfg.ImageTemplate.ExpandOrDie, - } - openshiftConfig, err := origin.BuildMasterConfig(openshiftConfigParameters) - if err != nil { - return nil, err - } - - return openshiftConfig, nil -} - -func (cfg Config) BuildAuthConfig() (*origin.AuthConfig, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - masterPublicAddr, err := cfg.GetMasterPublicAddress() - if err != nil { - return nil, err - } - assetPublicAddr, err := cfg.GetAssetPublicAddress() - if err != nil { - return nil, err - } - - apiServerCAs, err := cfg.GetAPIServerCertCAPool() - if err != nil { - return nil, err - } - - // Build the list of valid redirect_uri prefixes for a login using the openshift-web-console client to redirect to - // TODO: allow configuring this - // TODO: remove hard-coding of development UI server - assetPublicAddresses := []string{assetPublicAddr.String(), "http://localhost:9000", "https://localhost:9000"} - - etcdHelper, err := cfg.newOpenShiftEtcdHelper() - if err != nil { - return nil, fmt.Errorf("Error setting up server storage: %v", err) - } - // Default to a session authenticator (for browsers), and a basicauth authenticator (for clients responding to WWW-Authenticate challenges) - defaultAuthRequestHandlers := strings.Join([]string{ - string(origin.AuthRequestHandlerSession), - string(origin.AuthRequestHandlerBasicAuth), - }, ",") - - ret := &origin.AuthConfig{ - MasterAddr: masterAddr.String(), - MasterPublicAddr: masterPublicAddr.String(), - AssetPublicAddresses: assetPublicAddresses, - MasterRoots: apiServerCAs, - EtcdHelper: etcdHelper, - - // Max token ages - AuthorizeTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_AUTHORIZE_TOKEN_MAX_AGE_SECONDS", 300, 1), - AccessTokenMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_ACCESS_TOKEN_MAX_AGE_SECONDS", 3600, 1), - // Handlers - AuthRequestHandlers: origin.ParseAuthRequestHandlerTypes(env("OPENSHIFT_OAUTH_REQUEST_HANDLERS", defaultAuthRequestHandlers)), - AuthHandler: origin.AuthHandlerType(env("OPENSHIFT_OAUTH_HANDLER", string(origin.AuthHandlerLogin))), - GrantHandler: origin.GrantHandlerType(env("OPENSHIFT_OAUTH_GRANT_HANDLER", string(origin.GrantHandlerAuto))), - // RequestHeader config - RequestHeaders: strings.Split(env("OPENSHIFT_OAUTH_REQUEST_HEADERS", "X-Remote-User"), ","), - RequestHeaderCAFile: GetOAuthRequestHeaderCAFile(), - // Session config (default to unknowable secret) - SessionSecrets: []string{env("OPENSHIFT_OAUTH_SESSION_SECRET", uuid.NewUUID().String())}, - SessionMaxAgeSeconds: envInt("OPENSHIFT_OAUTH_SESSION_MAX_AGE_SECONDS", 300, 1), - SessionName: env("OPENSHIFT_OAUTH_SESSION_NAME", "ssn"), - // Password config - PasswordAuth: origin.PasswordAuthType(env("OPENSHIFT_OAUTH_PASSWORD_AUTH", string(origin.PasswordAuthAnyPassword))), - BasicAuthURL: env("OPENSHIFT_OAUTH_BASIC_AUTH_URL", ""), - HTPasswdFile: env("OPENSHIFT_OAUTH_HTPASSWD_FILE", ""), - // Token config - TokenStore: origin.TokenStoreType(env("OPENSHIFT_OAUTH_TOKEN_STORE", string(origin.TokenStoreOAuth))), - TokenFilePath: env("OPENSHIFT_OAUTH_TOKEN_FILE_PATH", ""), - // Google config - GoogleClientID: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_ID", ""), - GoogleClientSecret: env("OPENSHIFT_OAUTH_GOOGLE_CLIENT_SECRET", ""), - // GitHub config - GithubClientID: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_ID", ""), - GithubClientSecret: env("OPENSHIFT_OAUTH_GITHUB_CLIENT_SECRET", ""), - } - - return ret, nil - -} - -func GetOAuthRequestHeaderCAFile() string { - return env("OPENSHIFT_OAUTH_REQUEST_HEADER_CA_FILE", "") -} - -func (cfg Config) newCA() (*crypto.CA, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - - // Bootstrap CA - // TODO: store this (or parts of this) in etcd? - ca, err := crypto.InitCA(cfg.CertDir, fmt.Sprintf("%s@%d", masterAddr.Host, time.Now().Unix())) - if err != nil { - return nil, fmt.Errorf("Unable to configure certificate authority: %v", err) - } - - return ca, nil -} - -// GetAPIClientCertCAPool returns the cert pool used to validate client certificates to the API server -func (cfg Config) GetAPIClientCertCAPool() (*x509.CertPool, error) { - certs, err := cfg.getAPIClientCertCAs() - if err != nil { - return nil, err - } - roots := x509.NewCertPool() - for _, root := range certs { - roots.AddCert(root) - } - return roots, nil -} - -// GetClientCertCAPool returns a cert pool containing all client CAs that could be presented (union of API and OAuth) -func (cfg Config) GetClientCertCAPool() (*x509.CertPool, error) { - roots := x509.NewCertPool() - - // Add CAs for OAuth - certs, err := cfg.getOAuthClientCertCAs() - if err != nil { - return nil, err - } - for _, root := range certs { - roots.AddCert(root) - } - - // Add CAs for API - certs, err = cfg.getAPIClientCertCAs() - if err != nil { - return nil, err - } - for _, root := range certs { - roots.AddCert(root) - } - - return roots, nil -} - -// GetAPIServerCertCAPool returns the cert pool containing the roots for the API server cert -func (cfg Config) GetAPIServerCertCAPool() (*x509.CertPool, error) { - ca, err := cfg.newCA() - if err != nil { - return nil, err - } - roots := x509.NewCertPool() - for _, root := range ca.Config.Roots { - roots.AddCert(root) - } - return roots, nil -} - -func (cfg Config) getOAuthClientCertCAs() ([]*x509.Certificate, error) { - caFile := GetOAuthRequestHeaderCAFile() - if len(caFile) == 0 { - return nil, nil - } - caPEMBlock, err := ioutil.ReadFile(caFile) - if err != nil { - return nil, err - } - certs, err := crypto.CertsFromPEM(caPEMBlock) - if err != nil { - return nil, fmt.Errorf("Error reading %s: %s", caFile, err) - } - return certs, nil -} - -func (cfg Config) getAPIClientCertCAs() ([]*x509.Certificate, error) { - ca, err := cfg.newCA() - if err != nil { - return nil, err - } - return ca.Config.Roots, nil -} - -func (cfg Config) GetServerCertHostnames() ([]string, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - masterPublicAddr, err := cfg.GetMasterPublicAddress() - if err != nil { - return nil, err - } - kubePublicAddr, err := cfg.GetKubernetesPublicAddress() - if err != nil { - return nil, err - } - assetPublicAddr, err := cfg.GetAssetPublicAddress() - if err != nil { - return nil, err - } - - // 172.17.42.1 enables the router to call back out to the master - // TODO: Remove 172.17.42.1 once we can figure out how to validate the master's cert from inside a pod, or tell pods the real IP for the master - allHostnames := util.NewStringSet("localhost", "127.0.0.1", "172.17.42.1", masterAddr.Host, masterPublicAddr.Host, kubePublicAddr.Host, assetPublicAddr.Host) - certHostnames := util.StringSet{} - for hostname := range allHostnames { - if host, _, err := net.SplitHostPort(hostname); err == nil { - // add the hostname without the port - certHostnames.Insert(host) - } else { - // add the originally specified hostname - certHostnames.Insert(hostname) - } - } - - return certHostnames.List(), nil -} - -func (cfg Config) GetMasterCert() (certFile string, keyFile string, err error) { - ca, err := cfg.newCA() - if err != nil { - return "", "", err - } - - certHostnames, err := cfg.GetServerCertHostnames() - if err != nil { - return "", "", err - } - - serverCert, err := ca.MakeServerCert("master", certHostnames) - if err != nil { - return "", "", err - } - - return serverCert.CertFile, serverCert.KeyFile, nil -} - -func (cfg Config) GetAssetCert() (certFile string, keyFile string, err error) { - ca, err := cfg.newCA() - if err != nil { - return "", "", err - } - - certHostnames, err := cfg.GetServerCertHostnames() - if err != nil { - return "", "", err - } - - serverCert, err := ca.MakeServerCert("master", certHostnames) - if err != nil { - return "", "", err - } - - return serverCert.CertFile, serverCert.KeyFile, nil -} - -func (cfg Config) newClientConfigTemplate() (*clientcmdapi.Config, error) { - masterAddr, err := cfg.GetMasterAddress() - if err != nil { - return nil, err - } - masterPublicAddr, err := cfg.GetMasterPublicAddress() - if err != nil { - return nil, err - } - - return &clientcmdapi.Config{ - Clusters: map[string]clientcmdapi.Cluster{ - "master": {Server: masterAddr.String()}, - "public-master": {Server: masterPublicAddr.String()}, - }, - Contexts: map[string]clientcmdapi.Context{ - "master": {Cluster: "master"}, - "public-master": {Cluster: "public-master"}, - }, - CurrentContext: "master", - }, nil -} - -func (cfg Config) GetKubeClient() (*kclient.Client, *kclient.Config, error) { - var err error - var kubeClientConfig *kclient.Config - - // if we're starting an all in one, make credentials for a kube client. - if cfg.StartKube { - kubeClientConfig, err = cfg.MintSystemClientCert("kube-client") - if err != nil { - return nil, nil, err - } - - } else { - // Get the kubernetes address we're using - kubeAddr, err := cfg.GetKubernetesAddress() - if err != nil { - return nil, nil, err - } - - // Try to get the kubeconfig - kubeCfg, ok, err := cfg.GetExternalKubernetesClientConfig() - if err != nil { - return nil, nil, err - } - if !ok { - // No kubeconfig was provided, so just make one that points at the specified host - // It probably won't work (since it has no auth), but they'll get to see failures logged - kubeCfg = &kclient.Config{Host: kubeAddr.String()} - } - - // Ensure the kubernetes address matches the one in the config - if kubeAddr.String() != kubeCfg.Host { - return nil, nil, fmt.Errorf("The Kubernetes server (%s) must match the server in the provided kubeconfig (%s)", kubeAddr.String(), kubeCfg.Host) - } - - kubeClientConfig = kubeCfg - } - - kubeClient, err := kclient.New(kubeClientConfig) - if err != nil { - return nil, nil, err - } - - return kubeClient, kubeClientConfig, nil -} - -func (cfg Config) GetOpenshiftClient() (*osclient.Client, *kclient.Config, error) { - clientConfig, err := cfg.MintSystemClientCert("openshift-client") - if err != nil { - return nil, nil, err - } - - client, err := osclient.New(clientConfig) - if err != nil { - return nil, nil, err - } - - return client, clientConfig, nil -} - -func (cfg Config) GetOpenshiftDeployerClientConfig() (*kclient.Config, error) { - clientConfig, err := cfg.MintSystemClientCert("openshift-deployer", "system:deployers") - if err != nil { - return nil, err - } - - return clientConfig, nil -} - -// known certs: -// openshiftClientUser := &user.DefaultInfo{Name: "system:openshift-client"} -// openshiftDeployerUser := &user.DefaultInfo{Name: "system:openshift-deployer", Groups: []string{"system:deployers"}} -// adminUser := &user.DefaultInfo{Name: "system:admin", Groups: []string{"system:cluster-admins"}} -// kubeClientUser := &user.DefaultInfo{Name: "system:kube-client"} -// // One for each node in cfg.GetNodeList() -func (cfg Config) MintSystemClientCert(username string, groups ...string) (*kclient.Config, error) { - ca, err := cfg.newCA() - if err != nil { - return nil, err - } - clientConfigTemplate, err := cfg.newClientConfigTemplate() - if err != nil { - return nil, err - } - - qualifiedUsername := "system:" + username - user := &user.DefaultInfo{Name: qualifiedUsername, Groups: groups} - config, err := ca.MakeClientConfig(username, user, *clientConfigTemplate) - if err != nil { - return nil, err - } - - return &config, nil -} - -func (cfg Config) MintNodeCerts() error { - for _, node := range cfg.NodeList { - username := "node-" + node - if _, err := cfg.MintSystemClientCert(username, "system:nodes"); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go deleted file mode 100644 index d8735acf2e02..000000000000 --- a/pkg/cmd/server/start.go +++ /dev/null @@ -1,229 +0,0 @@ -package server - -import ( - "fmt" - "net" - "net/http" - "net/url" - "os" - "strconv" - "strings" - - kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" - "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" - "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" - "github.com/coreos/go-systemd/daemon" - "github.com/golang/glog" - "github.com/openshift/origin/pkg/cmd/server/etcd" - "github.com/openshift/origin/pkg/cmd/server/kubernetes" - "github.com/openshift/origin/pkg/cmd/server/origin" -) - -func (cfg Config) startMaster() error { - // Allow privileged containers - // TODO: make this configurable and not the default https://github.com/openshift/origin/issues/662 - capabilities.Initialize(capabilities.Capabilities{ - AllowPrivileged: true, - }) - - cfg.MintNodeCerts() - cfg.MintSystemClientCert("admin", "system:cluster-admins") - cfg.MintSystemClientCert("openshift-deployer", "system:deployers") - cfg.MintSystemClientCert("openshift-client") - if cfg.StartKube { - cfg.MintSystemClientCert("kube-client") - } - glog.Infof("Client certificates and .kubeconfig files generated in %v", cfg.CertDir) - - openshiftConfig, err := cfg.BuildOriginMasterConfig() - if err != nil { - return err - } - - // must start policy caching immediately - openshiftConfig.RunPolicyCache() - - authConfig, err := cfg.BuildAuthConfig() - if err != nil { - return err - } - - glog.Infof("Nodes: %v", cfg.NodeList) - - if strings.Contains(openshiftConfig.MasterAddr, "127.0.0.1") { - glog.Infof("WARNING: Your server is being advertised only to the host - containers will not be able to communicate with the master without a proxy") - } - - if cfg.StartKube { - kubeConfig, err := cfg.BuildKubernetesMasterConfig(openshiftConfig.RequestContextMapper, openshiftConfig.KubeClient()) - if err != nil { - return err - } - kubeConfig.EnsurePortalFlags() - - openshiftConfig.Run([]origin.APIInstaller{kubeConfig}, []origin.APIInstaller{authConfig}) - - kubeConfig.RunScheduler() - kubeConfig.RunReplicationController() - kubeConfig.RunEndpointController() - kubeConfig.RunMinionController() - kubeConfig.RunResourceQuotaManager() - - } else { - kubeAddr, err := cfg.GetKubernetesAddress() - if err != nil { - return err - } - proxy := &kubernetes.ProxyConfig{ - KubernetesAddr: kubeAddr, - ClientConfig: &openshiftConfig.KubeClientConfig, - } - - openshiftConfig.Run([]origin.APIInstaller{proxy}, []origin.APIInstaller{authConfig}) - } - - // TODO: recording should occur in individual components - record.StartRecording(openshiftConfig.KubeClient().Events(""), kapi.EventSource{Component: "master"}) - - glog.Infof("Using images from %q", openshiftConfig.ImageFor("")) - - openshiftConfig.RunDNSServer() - openshiftConfig.RunAssetServer() - openshiftConfig.RunBuildController() - openshiftConfig.RunBuildPodController() - openshiftConfig.RunBuildImageChangeTriggerController() - openshiftConfig.RunDeploymentController() - openshiftConfig.RunDeployerPodController() - openshiftConfig.RunDeploymentConfigController() - openshiftConfig.RunDeploymentConfigChangeController() - openshiftConfig.RunDeploymentImageChangeTriggerController() - openshiftConfig.RunProjectAuthorizationCache() - - return nil -} - -// run launches the appropriate startup modes or returns an error. -func (cfg Config) Start(args []string) error { - if cfg.WriteConfigOnly { - return nil - } - - switch { - case cfg.StartMaster && cfg.StartNode: - glog.Infof("Starting an OpenShift all-in-one, reachable at %s (etcd: %s)", cfg.MasterAddr.String(), cfg.EtcdAddr.String()) - if cfg.MasterPublicAddr.Provided { - glog.Infof("OpenShift master public address is %s", cfg.MasterPublicAddr.String()) - } - - case cfg.StartMaster: - glog.Infof("Starting an OpenShift master, reachable at %s (etcd: %s)", cfg.MasterAddr.String(), cfg.EtcdAddr.String()) - if cfg.MasterPublicAddr.Provided { - glog.Infof("OpenShift master public address is %s", cfg.MasterPublicAddr.String()) - } - - case cfg.StartNode: - glog.Infof("Starting an OpenShift node, connecting to %s", cfg.MasterAddr.String()) - - } - - if cfg.StartEtcd { - if err := cfg.RunEtcd(); err != nil { - return err - } - } - - if env("OPENSHIFT_PROFILE", "") == "web" { - go func() { - glog.Infof("Starting profiling endpoint at http://127.0.0.1:6060/debug/pprof/") - glog.Fatal(http.ListenAndServe("127.0.0.1:6060", nil)) - }() - } - - if cfg.StartMaster { - if err := cfg.startMaster(); err != nil { - return err - } - } - - if cfg.StartNode { - kubeClient, _, err := cfg.GetKubeClient() - if err != nil { - return err - } - - if !cfg.StartMaster { - // TODO: recording should occur in individual components - record.StartRecording(kubeClient.Events(""), kapi.EventSource{Component: "node"}) - } - - nodeConfig, err := cfg.BuildKubernetesNodeConfig() - if err != nil { - return err - } - - nodeConfig.EnsureVolumeDir() - nodeConfig.EnsureDocker(cfg.Docker) - nodeConfig.RunProxy() - nodeConfig.RunKubelet() - } - - daemon.SdNotify("READY=1") - select {} - - return nil -} - -func envInt(key string, defaultValue int32, minValue int32) int32 { - value, err := strconv.ParseInt(env(key, fmt.Sprintf("%d", defaultValue)), 10, 32) - if err != nil || int32(value) < minValue { - glog.Warningf("Invalid %s. Defaulting to %d.", key, defaultValue) - return defaultValue - } - return int32(value) -} - -// env returns an environment variable or a default value if not specified. -func env(key string, defaultValue string) string { - val := os.Getenv(key) - if len(val) == 0 { - return defaultValue - } - return val -} - -func (cfg Config) RunEtcd() error { - etcdAddr, err := cfg.GetEtcdAddress() - if err != nil { - return err - } - - etcdConfig := &etcd.Config{ - BindAddr: cfg.GetEtcdBindAddress(), - PeerBindAddr: cfg.GetEtcdPeerBindAddress(), - MasterAddr: etcdAddr.Host, - EtcdDir: cfg.EtcdDir, - } - - etcdConfig.Run() - - return nil -} - -func getHost(theURL url.URL) string { - host, _, err := net.SplitHostPort(theURL.Host) - if err != nil { - return theURL.Host - } - - return host -} - -func getPort(theURL url.URL) int { - _, port, err := net.SplitHostPort(theURL.Host) - if err != nil { - return 0 - } - - intport, _ := strconv.Atoi(port) - return intport -} diff --git a/pkg/cmd/server/start/bindaddr_arg.go b/pkg/cmd/server/start/bindaddr_arg.go new file mode 100644 index 000000000000..fd3e39886e0d --- /dev/null +++ b/pkg/cmd/server/start/bindaddr_arg.go @@ -0,0 +1,24 @@ +package start + +import ( + "github.com/spf13/pflag" + + "github.com/openshift/origin/pkg/cmd/flagtypes" +) + +// BindAddrArg is a struct that the command stores flag values into. +type BindAddrArg struct { + BindAddr flagtypes.Addr +} + +func BindBindAddrArg(args *BindAddrArg, flags *pflag.FlagSet, prefix string) { + flags.Var(&args.BindAddr, prefix+"listen", "The address to listen for connections on (host, host:port, or URL).") +} + +func NewDefaultBindAddrArg() *BindAddrArg { + config := &BindAddrArg{ + BindAddr: flagtypes.Addr{Value: "0.0.0.0:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + } + + return config +} diff --git a/pkg/cmd/server/start/certs_args.go b/pkg/cmd/server/start/certs_args.go new file mode 100644 index 000000000000..91781068d351 --- /dev/null +++ b/pkg/cmd/server/start/certs_args.go @@ -0,0 +1,20 @@ +package start + +import ( + "github.com/spf13/pflag" +) + +type CertArgs struct { + CertDir string + CreateCerts bool + OverwriteCerts bool +} + +func BindCertArgs(args *CertArgs, flags *pflag.FlagSet, prefix string) { + flags.BoolVar(&args.CreateCerts, prefix+"create-certs", true, "Create any missing certificates required for launch or for writing the config file.") + flags.StringVar(&args.CertDir, prefix+"cert-dir", "openshift.local.certificates", "The certificate data directory.") +} + +func NewDefaultCertArgs() *CertArgs { + return &CertArgs{CreateCerts: true} +} diff --git a/pkg/cmd/server/start/command_test.go b/pkg/cmd/server/start/command_test.go new file mode 100644 index 000000000000..9e6d5388d773 --- /dev/null +++ b/pkg/cmd/server/start/command_test.go @@ -0,0 +1,338 @@ +package start + +import ( + "io/ioutil" + "os" + "strconv" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" +) + +func TestCommandBindingListen(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--listen=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.BindAddrArg.BindAddr.Set(valueToSet) + + if expectedArgs.BindAddrArg.BindAddr.String() != actualCfg.BindAddrArg.BindAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.BindAddrArg.BindAddr.String(), actualCfg.BindAddrArg.BindAddr.String()) + } +} + +func TestCommandBindingMaster(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--master=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.MasterAddr.Set(valueToSet) + + if expectedArgs.MasterAddr.String() != actualCfg.MasterAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.MasterAddr.String(), actualCfg.MasterAddr.String()) + } +} + +func TestCommandBindingMasterPublic(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--public-master=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.MasterPublicAddr.Set(valueToSet) + + if expectedArgs.MasterPublicAddr.String() != actualCfg.MasterPublicAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.MasterPublicAddr.String(), actualCfg.MasterPublicAddr.String()) + } +} + +func TestCommandBindingEtcd(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--etcd=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.EtcdAddr.Set(valueToSet) + + if expectedArgs.EtcdAddr.String() != actualCfg.EtcdAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.EtcdAddr.String(), actualCfg.EtcdAddr.String()) + } +} + +func TestCommandBindingKubernetes(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--kubernetes=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.KubeConnectionArgs.KubernetesAddr.Set(valueToSet) + + if expectedArgs.KubeConnectionArgs.KubernetesAddr.String() != actualCfg.KubeConnectionArgs.KubernetesAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.KubeConnectionArgs.KubernetesAddr.String(), actualCfg.KubeConnectionArgs.KubernetesAddr.String()) + } +} + +func TestCommandBindingKubernetesPublic(t *testing.T) { + valueToSet := "http://example.org:9123" + actualCfg := executeMasterCommand([]string{"--public-kubernetes=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.KubernetesPublicAddr.Set(valueToSet) + + if expectedArgs.KubernetesPublicAddr.String() != actualCfg.KubernetesPublicAddr.String() { + t.Errorf("expected %v, got %v", expectedArgs.KubernetesPublicAddr.String(), actualCfg.KubernetesPublicAddr.String()) + } +} + +func TestCommandBindingPortalNet(t *testing.T) { + valueToSet := "192.168.0.0/16" + actualCfg := executeMasterCommand([]string{"--portal-net=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.PortalNet.Set(valueToSet) + + if expectedArgs.PortalNet.String() != actualCfg.PortalNet.String() { + t.Errorf("expected %v, got %v", expectedArgs.PortalNet.String(), actualCfg.PortalNet.String()) + } +} + +func TestCommandBindingImageTemplateFormat(t *testing.T) { + valueToSet := "some-format-string" + actualCfg := executeMasterCommand([]string{"--images=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.ImageFormatArgs.ImageTemplate.Format = valueToSet + + if expectedArgs.ImageFormatArgs.ImageTemplate.Format != actualCfg.ImageFormatArgs.ImageTemplate.Format { + t.Errorf("expected %v, got %v", expectedArgs.ImageFormatArgs.ImageTemplate.Format, actualCfg.ImageFormatArgs.ImageTemplate.Format) + } +} + +func TestCommandBindingImageLatest(t *testing.T) { + expectedArgs := NewDefaultMasterArgs() + + valueToSet := strconv.FormatBool(!expectedArgs.ImageFormatArgs.ImageTemplate.Latest) + actualCfg := executeMasterCommand([]string{"--latest-images=" + valueToSet}) + + expectedArgs.ImageFormatArgs.ImageTemplate.Latest = !expectedArgs.ImageFormatArgs.ImageTemplate.Latest + + if expectedArgs.ImageFormatArgs.ImageTemplate.Latest != actualCfg.ImageFormatArgs.ImageTemplate.Latest { + t.Errorf("expected %v, got %v", expectedArgs.ImageFormatArgs.ImageTemplate.Latest, actualCfg.ImageFormatArgs.ImageTemplate.Latest) + } +} + +func TestCommandBindingVolumeDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeNodeCommand([]string{"--volume-dir=" + valueToSet}) + + expectedArgs := NewDefaultNodeArgs() + expectedArgs.VolumeDir = valueToSet + + if expectedArgs.VolumeDir != actualCfg.VolumeDir { + t.Errorf("expected %v, got %v", expectedArgs.VolumeDir, actualCfg.VolumeDir) + } +} + +func TestCommandBindingEtcdDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeMasterCommand([]string{"--etcd-dir=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.EtcdDir = valueToSet + + if expectedArgs.EtcdDir != actualCfg.EtcdDir { + t.Errorf("expected %v, got %v", expectedArgs.EtcdDir, actualCfg.EtcdDir) + } +} + +func TestCommandBindingCertDir(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeMasterCommand([]string{"--cert-dir=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.CertArgs.CertDir = valueToSet + + if expectedArgs.CertArgs.CertDir != actualCfg.CertArgs.CertDir { + t.Errorf("expected %v, got %v", expectedArgs.CertArgs.CertDir, actualCfg.CertArgs.CertDir) + } +} + +func TestCommandBindingHostname(t *testing.T) { + valueToSet := "some-string" + actualCfg := executeNodeCommand([]string{"--hostname=" + valueToSet}) + + expectedArgs := NewDefaultNodeArgs() + expectedArgs.NodeName = valueToSet + + if expectedArgs.NodeName != actualCfg.NodeName { + t.Errorf("expected %v, got %v", expectedArgs.NodeName, actualCfg.NodeName) + } +} + +// AllInOne always adds the default hostname +func TestCommandBindingNodesForAllInOneAppend(t *testing.T) { + valueToSet := "first,second,third" + actualMasterCfg, actualNodeConfig := executeAllInOneCommand([]string{"--nodes=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + + stringList := util.StringList{} + stringList.Set(valueToSet + "," + strings.ToLower(actualNodeConfig.NodeName)) + expectedArgs.NodeList.Set(strings.Join(util.NewStringSet(stringList...).List(), ",")) + + if expectedArgs.NodeList.String() != actualMasterCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedArgs.NodeList, actualMasterCfg.NodeList) + } +} + +// AllInOne always adds the default hostname +func TestCommandBindingNodesForAllInOneAppendNoDupes(t *testing.T) { + valueToSet := "first,localhost,second,third" + actualMasterCfg, _ := executeAllInOneCommand([]string{"--nodes=" + valueToSet, "--hostname=LOCALHOST"}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.NodeList.Set(valueToSet) + + util.NewStringSet() + + if expectedArgs.NodeList.String() != actualMasterCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedArgs.NodeList, actualMasterCfg.NodeList) + } +} + +// AllInOne always adds the default hostname +func TestCommandBindingNodesDefaultingAllInOne(t *testing.T) { + actualMasterCfg, _ := executeAllInOneCommand([]string{}) + + expectedArgs := NewDefaultMasterArgs() + expectedNodeArgs := NewDefaultNodeArgs() + expectedArgs.NodeList.Set(strings.ToLower(expectedNodeArgs.NodeName)) + + if expectedArgs.NodeList.String() != actualMasterCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedArgs.NodeList, actualMasterCfg.NodeList) + } +} + +// explicit start master never modifies the NodeList +func TestCommandBindingNodesForMaster(t *testing.T) { + valueToSet := "first,second,third" + actualCfg := executeMasterCommand([]string{"master", "--nodes=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.NodeList.Set(valueToSet) + + if expectedArgs.NodeList.String() != actualCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedArgs.NodeList, actualCfg.NodeList) + } +} + +// explicit start master never modifies the NodeList +func TestCommandBindingNodesDefaultingMaster(t *testing.T) { + actualCfg := executeMasterCommand([]string{"master"}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.NodeList.Set("") + + if expectedArgs.NodeList.String() != actualCfg.NodeList.String() { + t.Errorf("expected %v, got %v", expectedArgs.NodeList, actualCfg.NodeList) + } +} + +func TestCommandBindingCors(t *testing.T) { + valueToSet := "first,second,third" + actualCfg := executeMasterCommand([]string{"--cors-allowed-origins=" + valueToSet}) + + expectedArgs := NewDefaultMasterArgs() + expectedArgs.CORSAllowedOrigins.Set(valueToSet) + + if expectedArgs.CORSAllowedOrigins.String() != actualCfg.CORSAllowedOrigins.String() { + t.Errorf("expected %v, got %v", expectedArgs.CORSAllowedOrigins, actualCfg.CORSAllowedOrigins) + } +} + +func executeMasterCommand(args []string) *MasterArgs { + fakeConfigFile, _ := ioutil.TempFile("", "") + defer os.Remove(fakeConfigFile.Name()) + + argsToUse := make([]string, 0, 4+len(args)) + argsToUse = append(argsToUse, "master") + argsToUse = append(argsToUse, args...) + argsToUse = append(argsToUse, "--write-config") + argsToUse = append(argsToUse, "--create-certs=false") + argsToUse = append(argsToUse, "--config="+fakeConfigFile.Name()) + + root := &cobra.Command{ + Use: "openshift", + Short: "test", + Long: "", + Run: func(c *cobra.Command, args []string) { + c.Help() + }, + } + + openshiftStartCommand, cfg := NewCommandStartMaster() + root.AddCommand(openshiftStartCommand) + root.SetArgs(argsToUse) + root.Execute() + + return cfg.MasterArgs +} + +func executeAllInOneCommand(args []string) (*MasterArgs, *NodeArgs) { + fakeMasterConfigFile, _ := ioutil.TempFile("", "") + defer os.Remove(fakeMasterConfigFile.Name()) + fakeNodeConfigFile, _ := ioutil.TempFile("", "") + defer os.Remove(fakeNodeConfigFile.Name()) + + argsToUse := make([]string, 0, 4+len(args)) + argsToUse = append(argsToUse, "start") + argsToUse = append(argsToUse, args...) + argsToUse = append(argsToUse, "--write-config") + argsToUse = append(argsToUse, "--create-certs=false") + argsToUse = append(argsToUse, "--master-config="+fakeMasterConfigFile.Name()) + argsToUse = append(argsToUse, "--node-config="+fakeNodeConfigFile.Name()) + + root := &cobra.Command{ + Use: "openshift", + Short: "test", + Long: "", + Run: func(c *cobra.Command, args []string) { + c.Help() + }, + } + + openshiftStartCommand, cfg := NewCommandStartAllInOne() + root.AddCommand(openshiftStartCommand) + root.SetArgs(argsToUse) + root.Execute() + + return cfg.MasterArgs, cfg.NodeArgs +} + +func executeNodeCommand(args []string) *NodeArgs { + fakeConfigFile, _ := ioutil.TempFile("", "") + defer os.Remove(fakeConfigFile.Name()) + + argsToUse := make([]string, 0, 4+len(args)) + argsToUse = append(argsToUse, "node") + argsToUse = append(argsToUse, args...) + argsToUse = append(argsToUse, "--write-config") + argsToUse = append(argsToUse, "--create-certs=false") + argsToUse = append(argsToUse, "--config="+fakeConfigFile.Name()) + + root := &cobra.Command{ + Use: "openshift", + Short: "test", + Long: "", + Run: func(c *cobra.Command, args []string) { + c.Help() + }, + } + + openshiftStartCommand, cfg := NewCommandStartNode() + root.AddCommand(openshiftStartCommand) + root.SetArgs(argsToUse) + root.Execute() + + return cfg.NodeArgs +} diff --git a/pkg/cmd/server/config_test.go b/pkg/cmd/server/start/config_test.go similarity index 61% rename from pkg/cmd/server/config_test.go rename to pkg/cmd/server/start/config_test.go index a175203f9a6b..05ad88b18ca4 100644 --- a/pkg/cmd/server/config_test.go +++ b/pkg/cmd/server/start/config_test.go @@ -1,4 +1,4 @@ -package server +package start import ( "testing" @@ -11,10 +11,10 @@ import ( func TestMasterPublicAddressDefaulting(t *testing.T) { expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) - actual, err := cfg.GetMasterPublicAddress() + actual, err := masterArgs.GetMasterPublicAddress() if err != nil { t.Errorf("unexpected error: %v", err) } @@ -26,11 +26,11 @@ func TestMasterPublicAddressDefaulting(t *testing.T) { func TestMasterPublicAddressExplicit(t *testing.T) { expected := "http://external.com:12445" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set("http://internal.com:9012") - cfg.MasterPublicAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set("http://internal.com:9012") + masterArgs.MasterPublicAddr.Set(expected) - actual, err := cfg.GetMasterPublicAddress() + actual, err := masterArgs.GetMasterPublicAddress() if err != nil { t.Errorf("unexpected error: %v", err) } @@ -43,10 +43,10 @@ func TestAssetPublicAddressDefaulting(t *testing.T) { master := "http://example.com:9011" expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(master) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(master) - actual, err := cfg.GetAssetPublicAddress() + actual, err := masterArgs.GetAssetPublicAddress() if err != nil { t.Errorf("unexpected error: %v", err) } @@ -59,11 +59,11 @@ func TestAssetPublicAddressExplicit(t *testing.T) { master := "http://example.com:9011" expected := "https://example.com:9014" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(master) - cfg.AssetPublicAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(master) + masterArgs.AssetPublicAddr.Set(expected) - actual, err := cfg.GetAssetPublicAddress() + actual, err := masterArgs.GetAssetPublicAddress() if err != nil { t.Errorf("unexpected error: %v", err) } @@ -76,10 +76,10 @@ func TestAssetBindAddressDefaulting(t *testing.T) { bind := "1.2.3.4:9011" expected := "1.2.3.4:9012" - cfg := NewDefaultConfig() - cfg.BindAddr.Set(bind) + masterArgs := NewDefaultMasterArgs() + masterArgs.BindAddrArg.BindAddr.Set(bind) - actual := cfg.GetAssetBindAddress() + actual := masterArgs.GetAssetBindAddress() if expected != actual { t.Errorf("expected %v, got %v", expected, actual) } @@ -89,11 +89,11 @@ func TestAssetBindAddressExplicit(t *testing.T) { bind := "1.2.3.4:9011" expected := "2.3.4.5:1234" - cfg := NewDefaultConfig() - cfg.BindAddr.Set(bind) - cfg.AssetBindAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.BindAddrArg.BindAddr.Set(bind) + masterArgs.AssetBindAddr.Set(expected) - actual := cfg.GetAssetBindAddress() + actual := masterArgs.GetAssetBindAddress() if expected != actual { t.Errorf("expected %v, got %v", expected, actual) } @@ -102,12 +102,12 @@ func TestAssetBindAddressExplicit(t *testing.T) { func TestKubernetesPublicAddressDefaultToKubernetesAddress(t *testing.T) { expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.KubernetesAddr.Set(expected) - cfg.MasterPublicAddr.Set("unexpectedpublicmaster") - cfg.MasterAddr.Set("unexpectedmaster") + masterArgs := NewDefaultMasterArgs() + masterArgs.KubeConnectionArgs.KubernetesAddr.Set(expected) + masterArgs.MasterPublicAddr.Set("unexpectedpublicmaster") + masterArgs.MasterAddr.Set("unexpectedmaster") - actual, err := cfg.GetKubernetesPublicAddress() + actual, err := masterArgs.GetKubernetesPublicAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -119,11 +119,11 @@ func TestKubernetesPublicAddressDefaultToKubernetesAddress(t *testing.T) { func TestKubernetesPublicAddressDefaultToPublicMasterAddress(t *testing.T) { expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterPublicAddr.Set(expected) - cfg.MasterAddr.Set("unexpectedmaster") + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterPublicAddr.Set(expected) + masterArgs.MasterAddr.Set("unexpectedmaster") - actual, err := cfg.GetKubernetesPublicAddress() + actual, err := masterArgs.GetKubernetesPublicAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -135,10 +135,10 @@ func TestKubernetesPublicAddressDefaultToPublicMasterAddress(t *testing.T) { func TestKubernetesPublicAddressDefaultToMasterAddress(t *testing.T) { expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) - actual, err := cfg.GetKubernetesPublicAddress() + actual, err := masterArgs.GetKubernetesPublicAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -150,13 +150,13 @@ func TestKubernetesPublicAddressDefaultToMasterAddress(t *testing.T) { func TestKubernetesPublicAddressExplicit(t *testing.T) { expected := "http://external.com:12445" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set("http://internal.com:9012") - cfg.KubernetesAddr.Set("http://internal.com:9013") - cfg.MasterPublicAddr.Set("http://internal.com:9014") - cfg.KubernetesPublicAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set("http://internal.com:9012") + masterArgs.KubeConnectionArgs.KubernetesAddr.Set("http://internal.com:9013") + masterArgs.MasterPublicAddr.Set("http://internal.com:9014") + masterArgs.KubernetesPublicAddr.Set(expected) - actual, err := cfg.GetKubernetesPublicAddress() + actual, err := masterArgs.GetKubernetesPublicAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -168,10 +168,11 @@ func TestKubernetesPublicAddressExplicit(t *testing.T) { func TestKubernetesAddressDefaulting(t *testing.T) { expected := "http://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) + masterAddr, _ := masterArgs.GetMasterAddress() - actual, err := cfg.GetKubernetesAddress() + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -183,11 +184,12 @@ func TestKubernetesAddressDefaulting(t *testing.T) { func TestKubernetesAddressExplicit(t *testing.T) { expected := "http://external.com:12445" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set("http://internal.com:9012") - cfg.KubernetesAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set("http://internal.com:9012") + masterArgs.KubeConnectionArgs.KubernetesAddr.Set(expected) + masterAddr, _ := masterArgs.GetMasterAddress() - actual, err := cfg.GetKubernetesAddress() + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -200,10 +202,10 @@ func TestEtcdAddressDefaulting(t *testing.T) { expected := "http://example.com:4001" master := "https://example.com:9012" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(master) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(master) - actual, err := cfg.GetEtcdAddress() + actual, err := masterArgs.GetEtcdAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -215,11 +217,11 @@ func TestEtcdAddressDefaulting(t *testing.T) { func TestEtcdAddressExplicit(t *testing.T) { expected := "http://external.com:12445" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set("http://internal.com:9012") - cfg.EtcdAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set("http://internal.com:9012") + masterArgs.EtcdAddr.Set(expected) - actual, err := cfg.GetEtcdAddress() + actual, err := masterArgs.GetEtcdAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -231,8 +233,8 @@ func TestEtcdAddressExplicit(t *testing.T) { func TestEtcdBindAddressDefault(t *testing.T) { expected := "0.0.0.0:4001" - cfg := NewDefaultConfig() - actual := cfg.GetEtcdBindAddress() + masterArgs := NewDefaultMasterArgs() + actual := masterArgs.GetEtcdBindAddress() if expected != actual { t.Fatalf("expected %v, got %v", expected, actual) } @@ -241,8 +243,8 @@ func TestEtcdBindAddressDefault(t *testing.T) { func TestEtcdPeerAddressDefault(t *testing.T) { expected := "0.0.0.0:7001" - cfg := NewDefaultConfig() - actual := cfg.GetEtcdPeerBindAddress() + masterArgs := NewDefaultMasterArgs() + actual := masterArgs.GetEtcdPeerBindAddress() if expected != actual { t.Fatalf("expected %v, got %v", expected, actual) } @@ -251,10 +253,10 @@ func TestEtcdPeerAddressDefault(t *testing.T) { func TestEtcdBindAddressDefaultToBind(t *testing.T) { expected := "1.2.3.4:4001" - cfg := NewDefaultConfig() - cfg.BindAddr.Set("https://1.2.3.4:8080") + masterArgs := NewDefaultMasterArgs() + masterArgs.BindAddrArg.BindAddr.Set("https://1.2.3.4:8080") - actual := cfg.GetEtcdBindAddress() + actual := masterArgs.GetEtcdBindAddress() if expected != actual { t.Fatalf("expected %v, got %v", expected, actual) } @@ -267,11 +269,10 @@ func TestMasterAddressDefaultingToBindValues(t *testing.T) { } expected := "http://" + defaultIP.String() + ":9012" - cfg := NewDefaultConfig() - cfg.StartMaster = true - cfg.BindAddr.Set("http://0.0.0.0:9012") + masterArgs := NewDefaultMasterArgs() + masterArgs.BindAddrArg.BindAddr.Set("http://0.0.0.0:9012") - actual, err := cfg.GetMasterAddress() + actual, err := masterArgs.GetMasterAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -283,10 +284,10 @@ func TestMasterAddressDefaultingToBindValues(t *testing.T) { func TestMasterAddressExplicit(t *testing.T) { expected := "http://external.com:12445" - cfg := NewDefaultConfig() - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) - actual, err := cfg.GetMasterAddress() + actual, err := masterArgs.GetMasterAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -298,60 +299,43 @@ func TestMasterAddressExplicit(t *testing.T) { func TestKubeClientForExternalKubernetesMasterWithNoConfig(t *testing.T) { expected := "https://localhost:8443" - cfg := NewDefaultConfig() - cfg.StartMaster = true - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) + masterAddr, _ := masterArgs.GetMasterAddress() - actual, err := cfg.GetKubernetesAddress() + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } if expected != actual.String() { t.Fatalf("expected %v, got %v", expected, actual) } - - _, config, err := cfg.GetKubeClient() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if expected != config.Host { - t.Fatalf("expected %v, got %v", expected, config.Host) - } } func TestKubeClientForNodeWithNoConfig(t *testing.T) { expected := "https://localhost:8443" - cfg := NewDefaultConfig() - cfg.StartNode = true - cfg.MasterAddr.Set(expected) + masterArgs := NewDefaultMasterArgs() + masterArgs.MasterAddr.Set(expected) + masterAddr, _ := masterArgs.GetMasterAddress() - actual, err := cfg.GetKubernetesAddress() + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } if expected != actual.String() { t.Fatalf("expected %v, got %v", expected, actual) } - - _, config, err := cfg.GetKubeClient() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if expected != config.Host { - t.Fatalf("expected %v, got %v", expected, config.Host) - } } func TestKubeClientForExternalKubernetesMasterWithConfig(t *testing.T) { expectedServer := "https://some-other-server:1234" expectedUser := "myuser" - cfg := NewDefaultConfig() - cfg.StartMaster = true - cfg.ClientConfigLoadingRules, cfg.ClientConfig = makeKubeconfig(expectedServer, expectedUser) + masterArgs := NewDefaultMasterArgs() + masterArgs.KubeConnectionArgs.ClientConfigLoadingRules, masterArgs.KubeConnectionArgs.ClientConfig = makeKubeconfig(expectedServer, expectedUser) - actualPublic, err := cfg.GetKubernetesPublicAddress() + actualPublic, err := masterArgs.GetKubernetesPublicAddress() if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -359,132 +343,87 @@ func TestKubeClientForExternalKubernetesMasterWithConfig(t *testing.T) { t.Fatalf("expected %v, got %v", expectedServer, actualPublic) } - actual, err := cfg.GetKubernetesAddress() + masterAddr, _ := masterArgs.GetMasterAddress() + + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } if expectedServer != actual.String() { t.Fatalf("expected %v, got %v", expectedServer, actual) } - - _, config, err := cfg.GetKubeClient() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if config.Host != expectedServer { - t.Fatalf("expected %v, got %v", expectedServer, config.Host) - } - if config.Username != expectedUser { - t.Fatalf("expected %v, got %v", expectedUser, config.Username) - } } func TestKubeClientForNodeWithConfig(t *testing.T) { expectedServer := "https://some-other-server:1234" expectedUser := "myuser" - cfg := NewDefaultConfig() - cfg.StartNode = true - cfg.ClientConfigLoadingRules, cfg.ClientConfig = makeKubeconfig(expectedServer, expectedUser) + nodeArgs := NewDefaultNodeArgs() + nodeArgs.KubeConnectionArgs.ClientConfigLoadingRules, nodeArgs.KubeConnectionArgs.ClientConfig = makeKubeconfig(expectedServer, expectedUser) - actualPublic, err := cfg.GetKubernetesPublicAddress() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if expectedServer != actualPublic.String() { - t.Fatalf("expected %v, got %v", expectedServer, actualPublic) - } - - actual, err := cfg.GetKubernetesAddress() + actual, err := nodeArgs.KubeConnectionArgs.GetKubernetesAddress(nil) if err != nil { t.Fatalf("unexpected error: %v", err) } if expectedServer != actual.String() { t.Fatalf("expected %v, got %v", expectedServer, actual) } - - _, config, err := cfg.GetKubeClient() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if config.Host != expectedServer { - t.Fatalf("expected %v, got %v", expectedServer, config.Host) - } - if config.Username != expectedUser { - t.Fatalf("expected %v, got %v", expectedUser, config.Username) - } } func TestKubeClientForExternalKubernetesMasterWithErrorKubeconfig(t *testing.T) { - cfg := NewDefaultConfig() - cfg.StartMaster = true - cfg.ClientConfigLoadingRules, cfg.ClientConfig = makeErrorKubeconfig() + masterArgs := NewDefaultMasterArgs() + masterArgs.KubeConnectionArgs.ClientConfigLoadingRules, masterArgs.KubeConnectionArgs.ClientConfig = makeErrorKubeconfig() // GetKubernetesPublicAddress hits the invalid kubeconfig in the fallback chain - _, err := cfg.GetKubernetesPublicAddress() + _, err := masterArgs.GetKubernetesPublicAddress() if err == nil { t.Fatalf("expected error, got none") } // GetKubernetesAddress hits the invalid kubeconfig in the fallback chain - _, err = cfg.GetKubernetesAddress() + masterAddr, _ := masterArgs.GetMasterAddress() + _, err = masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err == nil { t.Fatalf("expected error, got none") } - - // Should not get a client - if _, _, err = cfg.GetKubeClient(); err == nil { - t.Fatalf("expected error, got none") - } } func TestKubeClientForExternalKubernetesMasterWithConflictingKubernetesAddress(t *testing.T) { expectedServer := "https://some-other-server:1234" expectedUser := "myuser" - cfg := NewDefaultConfig() - cfg.StartMaster = true + masterArgs := NewDefaultMasterArgs() // Explicitly set --kubernetes must match --kubeconfig or return an error - cfg.KubernetesAddr.Set(expectedServer) - cfg.ClientConfigLoadingRules, cfg.ClientConfig = makeKubeconfig("https://another-server:2345", expectedUser) + masterArgs.KubeConnectionArgs.KubernetesAddr.Set(expectedServer) + masterArgs.KubeConnectionArgs.ClientConfigLoadingRules, masterArgs.KubeConnectionArgs.ClientConfig = makeKubeconfig("https://another-server:2345", expectedUser) // GetKubernetesAddress returns the explicitly set address - actual, err := cfg.GetKubernetesAddress() + masterAddr, _ := masterArgs.GetMasterAddress() + actual, err := masterArgs.KubeConnectionArgs.GetKubernetesAddress(masterAddr) if err != nil { t.Fatalf("unexpected error: %v", err) } if expectedServer != actual.String() { t.Fatalf("expected %v, got %v", expectedServer, actual) } - - // Should not get a client that might let us send credentials to the wrong server - if _, _, err := cfg.GetKubeClient(); err == nil { - t.Fatalf("expected error, got none") - } } func TestKubeClientForNodeWithConflictingKubernetesAddress(t *testing.T) { expectedServer := "https://some-other-server:1234" expectedUser := "myuser" - cfg := NewDefaultConfig() - cfg.StartNode = true - cfg.KubernetesAddr.Set(expectedServer) - cfg.ClientConfigLoadingRules, cfg.ClientConfig = makeKubeconfig("https://another-server:2345", expectedUser) + nodeArgs := NewDefaultNodeArgs() + nodeArgs.KubeConnectionArgs.KubernetesAddr.Set(expectedServer) + nodeArgs.KubeConnectionArgs.ClientConfigLoadingRules, nodeArgs.KubeConnectionArgs.ClientConfig = makeKubeconfig("https://another-server:2345", expectedUser) // GetKubernetesAddress returns the explicitly set address - actualServer, err := cfg.GetKubernetesAddress() + actualServer, err := nodeArgs.KubeConnectionArgs.GetKubernetesAddress(nil) if err != nil { t.Fatalf("unexpected error: %v", err) } if expectedServer != actualServer.String() { t.Fatalf("expected %v, got %v", expectedServer, actualServer) } - - // Should not get a client that might let us send credentials to the wrong server - if _, _, err := cfg.GetKubeClient(); err == nil { - t.Fatalf("expected error, got none") - } } func makeEmptyKubeconfig() (clientcmd.ClientConfigLoadingRules, clientcmd.ClientConfig) { diff --git a/pkg/cmd/server/start/image_args.go b/pkg/cmd/server/start/image_args.go new file mode 100644 index 000000000000..2fdbdf27646a --- /dev/null +++ b/pkg/cmd/server/start/image_args.go @@ -0,0 +1,25 @@ +package start + +import ( + "github.com/spf13/pflag" + + "github.com/openshift/origin/pkg/cmd/util/variable" +) + +// OriginMasterArgs is a struct that the command stores flag values into. +type ImageFormatArgs struct { + ImageTemplate variable.ImageTemplate +} + +func BindImageFormatArgs(args *ImageFormatArgs, flags *pflag.FlagSet, prefix string) { + flags.StringVar(&args.ImageTemplate.Format, "images", args.ImageTemplate.Format, "When fetching images used by the cluster for important components, use this format on both master and nodes. The latest release will be used by default.") + flags.BoolVar(&args.ImageTemplate.Latest, "latest-images", args.ImageTemplate.Latest, "If true, attempt to use the latest images for the cluster instead of the latest release.") +} + +func NewDefaultImageFormatArgs() *ImageFormatArgs { + config := &ImageFormatArgs{ + ImageTemplate: variable.NewDefaultImageTemplate(), + } + + return config +} diff --git a/pkg/cmd/server/start/kube_connection_args.go b/pkg/cmd/server/start/kube_connection_args.go new file mode 100644 index 000000000000..660d8443d59c --- /dev/null +++ b/pkg/cmd/server/start/kube_connection_args.go @@ -0,0 +1,72 @@ +package start + +import ( + "errors" + "net/url" + + "github.com/spf13/pflag" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" + + "github.com/openshift/origin/pkg/cmd/flagtypes" +) + +type KubeConnectionArgs struct { + KubernetesAddr flagtypes.Addr + + // ClientConfig is used when connecting to Kubernetes from the master, or + // when connecting to the master from a detached node. If StartKube is true, + // this value is not used. + ClientConfig clientcmd.ClientConfig + // ClientConfigLoadingRules is the ruleset used to load the client config. + // Only the CommandLinePath is expected to be used. + ClientConfigLoadingRules clientcmd.ClientConfigLoadingRules + + CertArgs *CertArgs +} + +func BindKubeConnectionArgs(args *KubeConnectionArgs, flags *pflag.FlagSet, prefix string) { + flags.Var(&args.KubernetesAddr, prefix+"kubernetes", "The address of the Kubernetes server (host, host:port, or URL). If specified, no Kubernetes components will be started.") + flags.StringVar(&args.ClientConfigLoadingRules.CommandLinePath, prefix+"kubeconfig", "", "Path to the kubeconfig file to use for requests to the Kubernetes API.") +} + +func NewDefaultKubeConnectionArgs() *KubeConnectionArgs { + config := &KubeConnectionArgs{} + + config.KubernetesAddr = flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default() + config.ClientConfig = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&config.ClientConfigLoadingRules, &clientcmd.ConfigOverrides{}) + config.CertArgs = NewDefaultCertArgs() + + return config +} + +func (args KubeConnectionArgs) GetExternalKubernetesClientConfig() (*client.Config, bool, error) { + if len(args.ClientConfigLoadingRules.CommandLinePath) == 0 || args.ClientConfig == nil { + return nil, false, nil + } + clientConfig, err := args.ClientConfig.ClientConfig() + if err != nil { + return nil, false, err + } + return clientConfig, true, nil +} + +func (args KubeConnectionArgs) GetKubernetesAddress(defaultAddress *url.URL) (*url.URL, error) { + if args.KubernetesAddr.Provided { + return args.KubernetesAddr.URL, nil + } + + config, ok, err := args.GetExternalKubernetesClientConfig() + if err != nil { + return nil, err + } + if ok && len(config.Host) > 0 { + return url.Parse(config.Host) + } + + if defaultAddress == nil { + return nil, errors.New("no default KubernetesAddress present") + } + return defaultAddress, nil +} diff --git a/pkg/cmd/server/start/master_args.go b/pkg/cmd/server/start/master_args.go new file mode 100644 index 000000000000..069bfdf8186b --- /dev/null +++ b/pkg/cmd/server/start/master_args.go @@ -0,0 +1,412 @@ +package start + +import ( + "fmt" + "net" + "net/url" + "strconv" + + "github.com/ghodss/yaml" + "github.com/spf13/pflag" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/cmd/flagtypes" + configapi "github.com/openshift/origin/pkg/cmd/server/api" + latestconfigapi "github.com/openshift/origin/pkg/cmd/server/api/latest" + "github.com/openshift/origin/pkg/cmd/server/certs" + cmdutil "github.com/openshift/origin/pkg/cmd/util" +) + +// MasterArgs is a struct that the command stores flag values into. It holds a partially complete set of parameters for starting the master +// This object should hold the common set values, but not attempt to handle all cases. The expected path is to use this object to create +// a fully specified config later on. If you need something not set here, then create a fully specified config file and pass that as argument +// to starting the master. +type MasterArgs struct { + MasterAddr flagtypes.Addr + EtcdAddr flagtypes.Addr + PortalNet flagtypes.IPNet + // addresses for external clients + MasterPublicAddr flagtypes.Addr + AssetPublicAddr flagtypes.Addr + KubernetesPublicAddr flagtypes.Addr + + // AssetBindAddr exposed for integration tests to set + AssetBindAddr flagtypes.Addr + // DNSBindAddr exposed for integration tests to set + DNSBindAddr flagtypes.Addr + + EtcdDir string + + NodeList util.StringList + + CORSAllowedOrigins util.StringList + + BindAddrArg *BindAddrArg + ImageFormatArgs *ImageFormatArgs + KubeConnectionArgs *KubeConnectionArgs + CertArgs *CertArgs +} + +// BindMasterArgs binds the options to the flags with prefix + default flag names +func BindMasterArgs(args *MasterArgs, flags *pflag.FlagSet, prefix string) { + flags.Var(&args.MasterAddr, prefix+"master", "The master address for use by OpenShift components (host, host:port, or URL). Scheme and port default to the --listen scheme and port.") + flags.Var(&args.MasterPublicAddr, prefix+"public-master", "The master address for use by public clients, if different (host, host:port, or URL). Defaults to same as --master.") + flags.Var(&args.EtcdAddr, prefix+"etcd", "The address of the etcd server (host, host:port, or URL). If specified, no built-in etcd will be started.") + flags.Var(&args.KubernetesPublicAddr, prefix+"public-kubernetes", "The Kubernetes server address for use by public clients, if different. (host, host:port, or URL). Defaults to same as --kubernetes.") + flags.Var(&args.PortalNet, prefix+"portal-net", "A CIDR notation IP range from which to assign portal IPs. This must not overlap with any IP ranges assigned to nodes for pods.") + + flags.StringVar(&args.EtcdDir, prefix+"etcd-dir", "openshift.local.etcd", "The etcd data directory.") + + flags.Var(&args.NodeList, prefix+"nodes", "The hostnames of each node. This currently must be specified up front. Comma delimited list") + flags.Var(&args.CORSAllowedOrigins, prefix+"cors-allowed-origins", "List of allowed origins for CORS, comma separated. An allowed origin can be a regular expression to support subdomain matching. CORS is enabled for localhost, 127.0.0.1, and the asset server by default.") +} + +// NewDefaultMasterArgs creates MasterArgs with sub-objects created and default values set. +func NewDefaultMasterArgs() *MasterArgs { + config := &MasterArgs{ + MasterAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + EtcdAddr: flagtypes.Addr{Value: "0.0.0.0:4001", DefaultScheme: "http", DefaultPort: 4001}.Default(), + PortalNet: flagtypes.DefaultIPNet("172.30.17.0/24"), + MasterPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + KubernetesPublicAddr: flagtypes.Addr{Value: "localhost:8443", DefaultScheme: "https", DefaultPort: 8443, AllowPrefix: true}.Default(), + AssetPublicAddr: flagtypes.Addr{Value: "localhost:8444", DefaultScheme: "https", DefaultPort: 8444, AllowPrefix: true}.Default(), + AssetBindAddr: flagtypes.Addr{Value: "0.0.0.0:8444", DefaultScheme: "https", DefaultPort: 8444, AllowPrefix: true}.Default(), + DNSBindAddr: flagtypes.Addr{Value: "0.0.0.0:53", DefaultScheme: "http", DefaultPort: 53, AllowPrefix: true}.Default(), + + BindAddrArg: NewDefaultBindAddrArg(), + ImageFormatArgs: NewDefaultImageFormatArgs(), + KubeConnectionArgs: NewDefaultKubeConnectionArgs(), + CertArgs: NewDefaultCertArgs(), + } + + return config +} + +// BuildSerializeableMasterConfig takes the MasterArgs (partially complete config) and uses them along with defaulting behavior to create the fully specified +// config object for starting the master +func (args MasterArgs) BuildSerializeableMasterConfig() (*configapi.MasterConfig, error) { + masterAddr, err := args.GetMasterAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := args.GetMasterPublicAddress() + if err != nil { + return nil, err + } + kubePublicAddr, err := args.GetKubernetesPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr, err := args.GetAssetPublicAddress() + if err != nil { + return nil, err + } + dnsBindAddr, err := args.GetDNSBindAddress() + if err != nil { + return nil, err + } + + corsAllowedOrigins := []string{} + corsAllowedOrigins = append(corsAllowedOrigins, args.CORSAllowedOrigins...) + // always include the all-in-one server's web console as an allowed CORS origin + // always include localhost as an allowed CORS origin + // always include master public address as an allowed CORS origin + for _, origin := range []string{assetPublicAddr.Host, masterPublicAddr.Host, "localhost", "127.0.0.1"} { + corsAllowedOrigins = append(corsAllowedOrigins, origin) + } + + etcdAddress, err := args.GetEtcdAddress() + if err != nil { + return nil, err + } + + var etcdConfig *configapi.EtcdConfig + if !args.EtcdAddr.Provided { + etcdConfig, err = args.BuildSerializeableEtcdConfig() + if err != nil { + return nil, err + } + } + var kubernetesMasterConfig *configapi.KubernetesMasterConfig + if !args.KubeConnectionArgs.KubernetesAddr.Provided && len(args.KubeConnectionArgs.ClientConfigLoadingRules.CommandLinePath) == 0 { + kubernetesMasterConfig, err = args.BuildSerializeableKubeMasterConfig() + if err != nil { + return nil, err + } + } + + config := &configapi.MasterConfig{ + ServingInfo: configapi.ServingInfo{ + BindAddress: args.BindAddrArg.BindAddr.URL.Host, + ServerCert: certs.DefaultMasterServingCertInfo(args.CertArgs.CertDir), + ClientCA: certs.DefaultRootCAFile(args.CertArgs.CertDir), + }, + CORSAllowedOrigins: corsAllowedOrigins, + + KubernetesMasterConfig: kubernetesMasterConfig, + EtcdConfig: etcdConfig, + + OAuthConfig: &configapi.OAuthConfig{ + ProxyCA: cmdutil.Env("OPENSHIFT_OAUTH_REQUEST_HEADER_CA_FILE", ""), + MasterURL: masterAddr.String(), + MasterPublicURL: masterPublicAddr.String(), + AssetPublicURL: assetPublicAddr.String(), + }, + + AssetConfig: &configapi.AssetConfig{ + ServingInfo: configapi.ServingInfo{ + BindAddress: args.GetAssetBindAddress(), + ServerCert: certs.DefaultAssetServingCertInfo(args.CertArgs.CertDir), + ClientCA: certs.DefaultRootCAFile(args.CertArgs.CertDir), + }, + + LogoutURI: cmdutil.Env("OPENSHIFT_LOGOUT_URI", ""), + MasterPublicURL: masterPublicAddr.String(), + PublicURL: assetPublicAddr.String(), + KubernetesPublicURL: kubePublicAddr.String(), + }, + + DNSConfig: &configapi.DNSConfig{ + BindAddress: dnsBindAddr.URL.Host, + }, + + MasterClients: configapi.MasterClients{ + DeployerKubeConfig: certs.DefaultKubeConfigFilename(args.CertArgs.CertDir, "openshift-deployer"), + OpenShiftLoopbackKubeConfig: certs.DefaultKubeConfigFilename(args.CertArgs.CertDir, "openshift-client"), + KubernetesKubeConfig: certs.DefaultKubeConfigFilename(args.CertArgs.CertDir, "kube-client"), + }, + + EtcdClientInfo: configapi.RemoteConnectionInfo{ + URL: etcdAddress.String(), + // TODO allow for https etcd + CA: "", + ClientCert: configapi.CertInfo{}, + }, + + MasterAuthorizationNamespace: "master", + OpenShiftSharedResourcesNamespace: "openshift", + + ImageConfig: configapi.ImageConfig{ + Format: args.ImageFormatArgs.ImageTemplate.Format, + Latest: args.ImageFormatArgs.ImageTemplate.Latest, + }, + } + + return config, nil +} + +// BuildSerializeableEtcdConfig creates a fully specified etcd startup configuration based on MasterArgs +func (args MasterArgs) BuildSerializeableEtcdConfig() (*configapi.EtcdConfig, error) { + etcdAddr, err := args.GetEtcdAddress() + if err != nil { + return nil, err + } + + config := &configapi.EtcdConfig{ + ServingInfo: configapi.ServingInfo{ + BindAddress: args.GetEtcdBindAddress(), + }, + PeerAddress: args.GetEtcdPeerBindAddress(), + MasterAddress: etcdAddr.Host, + StorageDir: args.EtcdDir, + } + + return config, nil +} + +// BuildSerializeableKubeMasterConfig creates a fully specified kubernetes master startup configuration based on MasterArgs +func (args MasterArgs) BuildSerializeableKubeMasterConfig() (*configapi.KubernetesMasterConfig, error) { + servicesSubnet := net.IPNet(args.PortalNet) + + config := &configapi.KubernetesMasterConfig{ + ServicesSubnet: servicesSubnet.String(), + StaticNodeNames: args.NodeList, + } + + return config, nil +} + +// GetServerCertHostnames returns the set of hostnames that any serving certificate for master needs to be valid for. +func (args MasterArgs) GetServerCertHostnames() (util.StringSet, error) { + masterAddr, err := args.GetMasterAddress() + if err != nil { + return nil, err + } + masterPublicAddr, err := args.GetMasterPublicAddress() + if err != nil { + return nil, err + } + kubePublicAddr, err := args.GetKubernetesPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr, err := args.GetAssetPublicAddress() + if err != nil { + return nil, err + } + + // 172.17.42.1 enables the router to call back out to the master + // TODO: Remove 172.17.42.1 once we can figure out how to validate the master's cert from inside a pod, or tell pods the real IP for the master + allHostnames := util.NewStringSet("localhost", "127.0.0.1", "172.17.42.1", masterAddr.Host, masterPublicAddr.Host, kubePublicAddr.Host, assetPublicAddr.Host) + certHostnames := util.StringSet{} + for hostname := range allHostnames { + if host, _, err := net.SplitHostPort(hostname); err == nil { + // add the hostname without the port + certHostnames.Insert(host) + } else { + // add the originally specified hostname + certHostnames.Insert(hostname) + } + } + + return certHostnames, nil +} + +// GetMasterAddress checks for an unset master address and then attempts to use the first +// public IPv4 non-loopback address registered on this host. +// TODO: make me IPv6 safe +func (args MasterArgs) GetMasterAddress() (*url.URL, error) { + if args.MasterAddr.Provided { + return args.MasterAddr.URL, nil + } + + // If the user specifies a bind address, and the master is not provided, use the bind port by default + port := args.MasterAddr.Port + if args.BindAddrArg.BindAddr.Provided { + port = args.BindAddrArg.BindAddr.Port + } + + // If the user specifies a bind address, and the master is not provided, use the bind scheme by default + scheme := args.MasterAddr.URL.Scheme + if args.BindAddrArg.BindAddr.Provided { + scheme = args.BindAddrArg.BindAddr.URL.Scheme + } + + addr := "" + if ip, err := cmdutil.DefaultLocalIP4(); err == nil { + addr = ip.String() + } else if err == cmdutil.ErrorNoDefaultIP { + addr = "127.0.0.1" + } else if err != nil { + return nil, fmt.Errorf("Unable to find a public IP address: %v", err) + } + + masterAddr := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port)) + return url.Parse(masterAddr) +} + +func (args MasterArgs) GetDNSBindAddress() (flagtypes.Addr, error) { + if args.DNSBindAddr.Provided { + return args.DNSBindAddr, nil + } + dnsAddr := flagtypes.Addr{Value: args.BindAddrArg.BindAddr.Host, DefaultPort: 53}.Default() + return dnsAddr, nil +} + +func (args MasterArgs) GetMasterPublicAddress() (*url.URL, error) { + if args.MasterPublicAddr.Provided { + return args.MasterPublicAddr.URL, nil + } + + return args.GetMasterAddress() +} + +func (args MasterArgs) GetEtcdBindAddress() string { + // Derive the etcd bind address by using the bind address and the default etcd port + return net.JoinHostPort(args.BindAddrArg.BindAddr.Host, strconv.Itoa(args.EtcdAddr.DefaultPort)) +} + +func (args MasterArgs) GetEtcdPeerBindAddress() string { + // Derive the etcd peer address by using the bind address and the default etcd peering port + return net.JoinHostPort(args.BindAddrArg.BindAddr.Host, "7001") +} + +func (args MasterArgs) GetEtcdAddress() (*url.URL, error) { + if args.EtcdAddr.Provided { + return args.EtcdAddr.URL, nil + } + + // Etcd should be reachable on the same address that the master is (for simplicity) + masterAddr, err := args.GetMasterAddress() + if err != nil { + return nil, err + } + + etcdAddr := net.JoinHostPort(getHost(*masterAddr), strconv.Itoa(args.EtcdAddr.DefaultPort)) + return url.Parse(args.EtcdAddr.DefaultScheme + "://" + etcdAddr) +} + +func (args MasterArgs) GetKubernetesPublicAddress() (*url.URL, error) { + if args.KubernetesPublicAddr.Provided { + return args.KubernetesPublicAddr.URL, nil + } + if args.KubeConnectionArgs.KubernetesAddr.Provided { + return args.KubeConnectionArgs.KubernetesAddr.URL, nil + } + config, ok, err := args.KubeConnectionArgs.GetExternalKubernetesClientConfig() + if err != nil { + return nil, err + } + if ok && len(config.Host) > 0 { + return url.Parse(config.Host) + } + + return args.GetMasterPublicAddress() +} + +func (args MasterArgs) GetAssetPublicAddress() (*url.URL, error) { + if args.AssetPublicAddr.Provided { + return args.AssetPublicAddr.URL, nil + } + // Derive the asset public address by incrementing the master public address port by 1 + // TODO: derive the scheme/port from the asset bind scheme/port once that is settable via the command line + t, err := args.GetMasterPublicAddress() + if err != nil { + return nil, err + } + assetPublicAddr := *t + assetPublicAddr.Host = net.JoinHostPort(getHost(assetPublicAddr), strconv.Itoa(getPort(assetPublicAddr)+1)) + + return &assetPublicAddr, nil +} + +func (args MasterArgs) GetAssetBindAddress() string { + if args.AssetBindAddr.Provided { + return args.AssetBindAddr.URL.Host + } + // Derive the asset bind address by incrementing the master bind address port by 1 + return net.JoinHostPort(args.BindAddrArg.BindAddr.Host, strconv.Itoa(args.BindAddrArg.BindAddr.Port+1)) +} + +func getHost(theURL url.URL) string { + host, _, err := net.SplitHostPort(theURL.Host) + if err != nil { + return theURL.Host + } + + return host +} + +func getPort(theURL url.URL) int { + _, port, err := net.SplitHostPort(theURL.Host) + if err != nil { + return 0 + } + + intport, _ := strconv.Atoi(port) + return intport +} + +// WriteMaster serializes the config to yaml. +func WriteMaster(config *configapi.MasterConfig) ([]byte, error) { + json, err := latestconfigapi.Codec.Encode(config) + if err != nil { + return nil, err + } + content, err := yaml.JSONToYAML(json) + if err != nil { + return nil, err + } + return content, nil +} diff --git a/pkg/cmd/server/start/node_args.go b/pkg/cmd/server/start/node_args.go new file mode 100644 index 000000000000..74a3f8cc62e6 --- /dev/null +++ b/pkg/cmd/server/start/node_args.go @@ -0,0 +1,127 @@ +package start + +import ( + "fmt" + "net" + "net/url" + "os/exec" + "strconv" + "strings" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports" + "github.com/ghodss/yaml" + "github.com/golang/glog" + "github.com/spf13/pflag" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" + latestconfigapi "github.com/openshift/origin/pkg/cmd/server/api/latest" + "github.com/openshift/origin/pkg/cmd/server/certs" + cmdutil "github.com/openshift/origin/pkg/cmd/util" +) + +// NodeArgs is a struct that the command stores flag values into. It holds a partially complete set of parameters for starting the master +// This object should hold the common set values, but not attempt to handle all cases. The expected path is to use this object to create +// a fully specified config later on. If you need something not set here, then create a fully specified config file and pass that as argument +// to starting the master. +type NodeArgs struct { + NodeName string + + AllowDisabledDocker bool + VolumeDir string + + DefaultKubernetesURL url.URL + ClusterDomain string + ClusterDNS net.IP + + BindAddrArg *BindAddrArg + ImageFormatArgs *ImageFormatArgs + KubeConnectionArgs *KubeConnectionArgs + CertArgs *CertArgs +} + +// BindNodeArgs binds the options to the flags with prefix + default flag names +func BindNodeArgs(args *NodeArgs, flags *pflag.FlagSet, prefix string) { + flags.StringVar(&args.VolumeDir, prefix+"volume-dir", "openshift.local.volumes", "The volume storage directory.") + // TODO rename this node-name and recommend hostname -f + flags.StringVar(&args.NodeName, prefix+"hostname", args.NodeName, "The hostname to identify this node with the master.") +} + +// NewDefaultNodeArgs creates NodeArgs with sub-objects created and default values set. +func NewDefaultNodeArgs() *NodeArgs { + hostname, err := defaultHostname() + if err != nil { + hostname = "localhost" + glog.Warningf("Unable to lookup hostname, using %q: %v", hostname, err) + } + + var dnsIP net.IP + if clusterDNS := cmdutil.Env("OPENSHIFT_DNS_ADDR", ""); len(clusterDNS) > 0 { + dnsIP = net.ParseIP(clusterDNS) + } + + return &NodeArgs{ + NodeName: hostname, + + ClusterDomain: cmdutil.Env("OPENSHIFT_DNS_DOMAIN", "local"), + ClusterDNS: dnsIP, + + BindAddrArg: NewDefaultBindAddrArg(), + ImageFormatArgs: NewDefaultImageFormatArgs(), + KubeConnectionArgs: NewDefaultKubeConnectionArgs(), + CertArgs: NewDefaultCertArgs(), + } +} + +// BuildSerializeableNodeConfig takes the NodeArgs (partially complete config) and uses them along with defaulting behavior to create the fully specified +// config object for starting the node +func (args NodeArgs) BuildSerializeableNodeConfig() (*configapi.NodeConfig, error) { + var dnsIP string + if len(args.ClusterDNS) > 0 { + dnsIP = args.ClusterDNS.String() + } + + config := &configapi.NodeConfig{ + NodeName: args.NodeName, + + ServingInfo: configapi.ServingInfo{ + BindAddress: net.JoinHostPort(args.BindAddrArg.BindAddr.Host, strconv.Itoa(ports.KubeletPort)), + ServerCert: certs.DefaultNodeServingCertInfo(args.CertArgs.CertDir, args.NodeName), + }, + + VolumeDirectory: args.VolumeDir, + NetworkContainerImage: args.ImageFormatArgs.ImageTemplate.ExpandOrDie("pod"), + AllowDisabledDocker: args.AllowDisabledDocker, + + DNSDomain: args.ClusterDomain, + DNSIP: dnsIP, + + MasterKubeConfig: certs.DefaultKubeConfigFilename(args.CertArgs.CertDir, "node-"+args.NodeName), + } + + return config, nil +} + +// WriteNode serializes the config to yaml. +func WriteNode(config *configapi.NodeConfig) ([]byte, error) { + json, err := latestconfigapi.Codec.Encode(config) + if err != nil { + return nil, err + } + content, err := yaml.JSONToYAML(json) + if err != nil { + return nil, err + } + return content, nil +} + +// defaultHostname returns the default hostname for this system. +func defaultHostname() (string, error) { + + // Note: We use exec here instead of os.Hostname() because we + // want the FQDN, and this is the easiest way to get it. + fqdn, err := exec.Command("hostname", "-f").Output() + if err != nil { + return "", fmt.Errorf("Couldn't determine hostname: %v", err) + } + return strings.TrimSpace(string(fqdn)), nil +} diff --git a/pkg/cmd/server/start/start_allinone.go b/pkg/cmd/server/start/start_allinone.go new file mode 100644 index 000000000000..c6f45a943a02 --- /dev/null +++ b/pkg/cmd/server/start/start_allinone.go @@ -0,0 +1,222 @@ +package start + +import ( + "errors" + "fmt" + "net" + "strings" + + "github.com/coreos/go-systemd/daemon" + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + "github.com/openshift/origin/pkg/cmd/server/certs" + + _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/admit" + _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/limitranger" + _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/namespace/exists" + _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcedefaults" + _ "github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/admission/resourcequota" +) + +type AllInOneOptions struct { + MasterArgs *MasterArgs + NodeArgs *NodeArgs + + WriteConfigOnly bool + MasterConfigFile string + NodeConfigFile string +} + +const longAllInOneCommandDesc = ` +Start an OpenShift all-in-one server + +This command helps you launch an OpenShift all-in-one server, which allows +you to run all of the components of an OpenShift system on a server with Docker. Running + + $ openshift start + +will start OpenShift listening on all interfaces, launch an etcd server to store persistent +data, and launch the Kubernetes system components. The server will run in the foreground until +you terminate the process. This command delegates to "openshift start master" and +"openshift start node". + + +Note: starting OpenShift without passing the --master address will attempt to find the IP +address that will be visible inside running Docker containers. This is not always successful, +so if you have problems tell OpenShift what public address it will be via --master=. + +You may also pass --etcd=
to connect to an external etcd server. + +You may also pass --kubeconfig= to connect to an external Kubernetes cluster. +` + +// NewCommandStartMaster provides a CLI handler for 'start' command +func NewCommandStartAllInOne() (*cobra.Command, *AllInOneOptions) { + options := &AllInOneOptions{} + + cmd := &cobra.Command{ + Use: "start", + Short: "Launch OpenShift All-In-One", + Long: longAllInOneCommandDesc, + Run: func(c *cobra.Command, args []string) { + if err := options.Complete(); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if err := options.StartAllInOne(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + + flags.BoolVar(&options.WriteConfigOnly, "write-config", false, "Indicates that the command should build the configuration from command-line arguments, write it to the locations specified by --master-config and --node-config, and exit.") + flags.StringVar(&options.MasterConfigFile, "master-config", "", "Location of the master configuration file to run from, or write to (when used with --write-config). When running from configuration files, all other command-line arguments are ignored.") + flags.StringVar(&options.NodeConfigFile, "node-config", "", "Location of the node configuration file to run from, or write to (when used with --write-config). When running from configuration files, all other command-line arguments are ignored.") + + masterArgs, nodeArgs, bindAddrArg, imageFormatArgs, kubeConnectionArgs, certArgs := GetAllInOneArgs() + options.MasterArgs, options.NodeArgs = masterArgs, nodeArgs + // by default, all-in-ones all disabled docker. Set it here so that if we allow it to be bound later, bindings take precendence + options.NodeArgs.AllowDisabledDocker = true + + BindMasterArgs(masterArgs, flags, "") + BindNodeArgs(nodeArgs, flags, "") + BindBindAddrArg(bindAddrArg, flags, "") + BindImageFormatArgs(imageFormatArgs, flags, "") + BindKubeConnectionArgs(kubeConnectionArgs, flags, "") + BindCertArgs(certArgs, flags, "") + + startMaster, _ := NewCommandStartMaster() + startNode, _ := NewCommandStartNode() + cmd.AddCommand(startMaster) + cmd.AddCommand(startNode) + + return cmd, options +} + +// GetAllInOneArgs makes sure that the node and master args that should be shared, are shared +func GetAllInOneArgs() (*MasterArgs, *NodeArgs, *BindAddrArg, *ImageFormatArgs, *KubeConnectionArgs, *CertArgs) { + masterArgs := NewDefaultMasterArgs() + nodeArgs := NewDefaultNodeArgs() + + bindAddrArg := NewDefaultBindAddrArg() + masterArgs.BindAddrArg = bindAddrArg + nodeArgs.BindAddrArg = bindAddrArg + + imageFormatArgs := NewDefaultImageFormatArgs() + masterArgs.ImageFormatArgs = imageFormatArgs + nodeArgs.ImageFormatArgs = imageFormatArgs + + kubeConnectionArgs := NewDefaultKubeConnectionArgs() + masterArgs.KubeConnectionArgs = kubeConnectionArgs + nodeArgs.KubeConnectionArgs = kubeConnectionArgs + + certArgs := NewDefaultCertArgs() + masterArgs.CertArgs = certArgs + nodeArgs.CertArgs = certArgs + kubeConnectionArgs.CertArgs = certArgs + + return masterArgs, nodeArgs, bindAddrArg, imageFormatArgs, kubeConnectionArgs, certArgs +} + +func (o AllInOneOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported for start") + } + if o.WriteConfigOnly { + if len(o.MasterConfigFile) == 0 { + return errors.New("--master-config is required if --write-config is true") + } + if len(o.NodeConfigFile) == 0 { + return errors.New("--node-config is required if --write-config is true") + } + } + + return nil +} + +func (o AllInOneOptions) Complete() error { + nodeList := util.NewStringSet(strings.ToLower(o.NodeArgs.NodeName)) + // take everything toLower + for _, s := range o.MasterArgs.NodeList { + nodeList.Insert(strings.ToLower(s)) + } + o.MasterArgs.NodeList = nodeList.List() + + o.NodeArgs.NodeName = strings.ToLower(o.NodeArgs.NodeName) + + return nil +} + +// StartAllInOne: +// 1. Creates the signer certificate if needed +// 2. Calls RunMaster +// 3. Calls RunNode +// 4. If only writing configs, it exits +// 5. Waits forever +func (o AllInOneOptions) StartAllInOne() error { + if !o.WriteConfigOnly { + glog.Infof("Starting an OpenShift all-in-one") + } + + // if either one of these wants to mint certs, make sure the signer is present BEFORE they start up to make sure they always share + if o.MasterArgs.CertArgs.CreateCerts || o.NodeArgs.CertArgs.CreateCerts { + signerOptions := &certs.CreateSignerCertOptions{ + CertFile: certs.DefaultCertFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + KeyFile: certs.DefaultKeyFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + SerialFile: certs.DefaultSerialFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + Name: certs.DefaultSignerName(), + } + + if _, err := signerOptions.CreateSignerCert(); err != nil { + return err + } + } + + masterOptions := MasterOptions{o.MasterArgs, o.WriteConfigOnly, o.MasterConfigFile} + + masterAddr, err := masterOptions.MasterArgs.GetMasterAddress() + if err != nil { + return nil + } + + // in the all-in-one, default kubernetes URL to the master's address + o.NodeArgs.DefaultKubernetesURL = *masterAddr + + // in the all-in-one, default ClusterDNS to the master's address + if host, _, err := net.SplitHostPort(masterAddr.Host); err == nil { + if ip := net.ParseIP(host); ip != nil { + o.NodeArgs.ClusterDNS = ip + } + } + + nodeOptions := NodeOptions{o.NodeArgs, o.WriteConfigOnly, o.NodeConfigFile} + + if err := masterOptions.RunMaster(); err != nil { + return err + } + + if err := nodeOptions.RunNode(); err != nil { + return err + } + + if o.WriteConfigOnly { + return nil + } + + daemon.SdNotify("READY=1") + select {} + + return nil +} diff --git a/pkg/cmd/server/start/start_master.go b/pkg/cmd/server/start/start_master.go new file mode 100644 index 000000000000..38a66251bfbf --- /dev/null +++ b/pkg/cmd/server/start/start_master.go @@ -0,0 +1,349 @@ +package start + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + _ "net/http/pprof" + "strings" + + "github.com/coreos/go-systemd/daemon" + "github.com/golang/glog" + "github.com/spf13/cobra" + + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" + configapilatest "github.com/openshift/origin/pkg/cmd/server/api/latest" + "github.com/openshift/origin/pkg/cmd/server/certs" + "github.com/openshift/origin/pkg/cmd/server/etcd" + "github.com/openshift/origin/pkg/cmd/server/kubernetes" + "github.com/openshift/origin/pkg/cmd/server/origin" + cmdutil "github.com/openshift/origin/pkg/cmd/util" +) + +type MasterOptions struct { + MasterArgs *MasterArgs + + WriteConfigOnly bool + ConfigFile string +} + +const longMasterCommandDesc = ` +Start an OpenShift master + +This command helps you launch an OpenShift master. Running + + $ openshift start master + +will start an OpenShift master listening on all interfaces, launch an etcd server to store +persistent data, and launch the Kubernetes system components. The server will run in the +foreground until you terminate the process. + +Note: starting OpenShift without passing the --master address will attempt to find the IP +address that will be visible inside running Docker containers. This is not always successful, +so if you have problems tell OpenShift what public address it will be via --master=. + +You may also pass an optional argument to the start command to start OpenShift in one of the +following roles: + + $ openshift start master --nodes= + + Launches the server and control plane for OpenShift. You may pass a list of the node + hostnames you want to use, or create nodes via the REST API or 'openshift kube'. + +You may also pass --etcd=
to connect to an external etcd server. + +You may also pass --kubeconfig= to connect to an external Kubernetes cluster. +` + +// NewCommandStartMaster provides a CLI handler for 'start' command +func NewCommandStartMaster() (*cobra.Command, *MasterOptions) { + options := &MasterOptions{} + + cmd := &cobra.Command{ + Use: "master", + Short: "Launch OpenShift master", + Long: longMasterCommandDesc, + Run: func(c *cobra.Command, args []string) { + if err := options.Complete(); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if err := options.StartMaster(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + + flags.BoolVar(&options.WriteConfigOnly, "write-config", false, "Indicates that the command should build the configuration from command-line arguments, write it to the location specified by --config, and exit.") + flags.StringVar(&options.ConfigFile, "config", "", "Location of the master configuration file to run from, or write to (when used with --write-config). When running from a configuration file, all other command-line arguments are ignored.") + + options.MasterArgs = NewDefaultMasterArgs() + // make sure that KubeConnectionArgs and MasterArgs use the same CertArgs for this command + options.MasterArgs.KubeConnectionArgs.CertArgs = options.MasterArgs.CertArgs + + BindMasterArgs(options.MasterArgs, flags, "") + BindBindAddrArg(options.MasterArgs.BindAddrArg, flags, "") + BindImageFormatArgs(options.MasterArgs.ImageFormatArgs, flags, "") + BindKubeConnectionArgs(options.MasterArgs.KubeConnectionArgs, flags, "") + BindCertArgs(options.MasterArgs.CertArgs, flags, "") + + return cmd, options +} + +func (o MasterOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported for start master") + } + if o.WriteConfigOnly { + if len(o.ConfigFile) == 0 { + return errors.New("--config is required if --write-config is true") + } + } + + return nil +} + +func (o MasterOptions) Complete() error { + nodeList := util.NewStringSet() + // take everything toLower + for _, s := range o.MasterArgs.NodeList { + nodeList.Insert(strings.ToLower(s)) + } + + o.MasterArgs.NodeList = nodeList.List() + + return nil +} + +// StartMaster calls RunMaster and then waits forever +func (o MasterOptions) StartMaster() error { + if err := o.RunMaster(); err != nil { + return err + } + + if o.WriteConfigOnly { + return nil + } + + daemon.SdNotify("READY=1") + select {} + + return nil +} + +// RunMaster takes the options and: +// 1. Creates certs if needed +// 2. Reads fully specified master config OR builds a fully specified master config from the args +// 3. Writes the fully specified master config and exits if needed +// 4. Starts the master based on the fully specified config +func (o MasterOptions) RunMaster() error { + startUsingConfigFile := !o.WriteConfigOnly && (len(o.ConfigFile) > 0) + mintCerts := o.MasterArgs.CertArgs.CreateCerts && !startUsingConfigFile + + if mintCerts { + if err := o.CreateCerts(); err != nil { + return nil + } + } + + var masterConfig *configapi.MasterConfig + var err error + if startUsingConfigFile { + masterConfig, err = ReadMasterConfig(o.ConfigFile) + } else { + masterConfig, err = o.MasterArgs.BuildSerializeableMasterConfig() + } + if err != nil { + return err + } + + if o.WriteConfigOnly { + content, err := WriteMaster(masterConfig) + if err != nil { + return err + } + if err := ioutil.WriteFile(o.ConfigFile, content, 0644); err != nil { + return err + } + return nil + } + + if err := StartMaster(masterConfig); err != nil { + return err + } + + return nil +} + +func (o MasterOptions) CreateCerts() error { + signerName := certs.DefaultSignerName() + hostnames, err := o.MasterArgs.GetServerCertHostnames() + if err != nil { + return err + } + mintAllCertsOptions := certs.CreateAllCertsOptions{ + CertDir: o.MasterArgs.CertArgs.CertDir, + SignerName: signerName, + Hostnames: hostnames.List(), + NodeList: o.MasterArgs.NodeList, + } + if err := mintAllCertsOptions.CreateAllCerts(); err != nil { + return err + } + + rootCAFile := certs.DefaultRootCAFile(o.MasterArgs.CertArgs.CertDir) + masterAddr, err := o.MasterArgs.GetMasterAddress() + if err != nil { + return err + } + publicMasterAddr, err := o.MasterArgs.GetMasterPublicAddress() + if err != nil { + return err + } + for _, clientCertInfo := range certs.DefaultClientCerts(o.MasterArgs.CertArgs.CertDir) { + createKubeConfigOptions := certs.CreateKubeConfigOptions{ + APIServerURL: masterAddr.String(), + PublicAPIServerURL: publicMasterAddr.String(), + APIServerCAFile: rootCAFile, + ServerNick: "master", + + CertFile: clientCertInfo.CertLocation.CertFile, + KeyFile: clientCertInfo.CertLocation.KeyFile, + UserNick: clientCertInfo.SubDir, + + KubeConfigFile: certs.DefaultKubeConfigFilename(o.MasterArgs.CertArgs.CertDir, clientCertInfo.SubDir), + } + + if _, err := createKubeConfigOptions.CreateKubeConfig(); err != nil { + return err + } + } + + return nil +} + +func ReadMasterConfig(filename string) (*configapi.MasterConfig, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + config := &configapi.MasterConfig{} + + if err := configapilatest.Codec.DecodeInto(data, config); err != nil { + return nil, err + } + return config, nil +} + +func StartMaster(openshiftMasterConfig *configapi.MasterConfig) error { + glog.Infof("Starting an OpenShift master, reachable at %s (etcd: %s)", openshiftMasterConfig.ServingInfo.BindAddress, openshiftMasterConfig.EtcdClientInfo.URL) + glog.Infof("OpenShift master public address is %s", openshiftMasterConfig.AssetConfig.MasterPublicURL) + + if openshiftMasterConfig.EtcdConfig != nil { + etcdConfig := &etcd.Config{ + BindAddr: openshiftMasterConfig.EtcdConfig.ServingInfo.BindAddress, + PeerBindAddr: openshiftMasterConfig.EtcdConfig.PeerAddress, + MasterAddr: openshiftMasterConfig.EtcdConfig.MasterAddress, + EtcdDir: openshiftMasterConfig.EtcdConfig.StorageDir, + } + + etcdConfig.Run() + } + + if cmdutil.Env("OPENSHIFT_PROFILE", "") == "web" { + go func() { + glog.Infof("Starting profiling endpoint at http://127.0.0.1:6060/debug/pprof/") + glog.Fatal(http.ListenAndServe("127.0.0.1:6060", nil)) + }() + } + + // Allow privileged containers + // TODO: make this configurable and not the default https://github.com/openshift/origin/issues/662 + capabilities.Initialize(capabilities.Capabilities{ + AllowPrivileged: true, + }) + + openshiftConfig, err := origin.BuildMasterConfig(*openshiftMasterConfig) + if err != nil { + return err + } + // must start policy caching immediately + openshiftConfig.RunPolicyCache() + + authConfig, err := origin.BuildAuthConfig(*openshiftMasterConfig) + if err != nil { + return err + } + + if openshiftMasterConfig.KubernetesMasterConfig != nil { + glog.Infof("Static Nodes: %v", openshiftMasterConfig.KubernetesMasterConfig.StaticNodeNames) + + kubeConfig, err := kubernetes.BuildKubernetesMasterConfig(*openshiftMasterConfig, openshiftConfig.RequestContextMapper, openshiftConfig.KubeClient()) + if err != nil { + return err + } + kubeConfig.EnsurePortalFlags() + + openshiftConfig.Run([]origin.APIInstaller{kubeConfig}, []origin.APIInstaller{authConfig}) + + kubeConfig.RunScheduler() + kubeConfig.RunReplicationController() + kubeConfig.RunEndpointController() + kubeConfig.RunMinionController() + kubeConfig.RunResourceQuotaManager() + + } else { + _, kubeConfig, err := configapi.GetKubeClient(openshiftMasterConfig.MasterClients.KubernetesKubeConfig) + if err != nil { + return err + } + + proxy := &kubernetes.ProxyConfig{ + ClientConfig: kubeConfig, + } + + openshiftConfig.Run([]origin.APIInstaller{proxy}, []origin.APIInstaller{authConfig}) + } + + // TODO: recording should occur in individual components + record.StartRecording(openshiftConfig.KubeClient().Events(""), kapi.EventSource{Component: "master"}) + + glog.Infof("Using images from %q", openshiftConfig.ImageFor("")) + + if openshiftMasterConfig.DNSConfig != nil { + openshiftConfig.RunDNSServer() + } + if openshiftMasterConfig.AssetConfig != nil { + openshiftConfig.RunAssetServer() + } + openshiftConfig.RunBuildController() + openshiftConfig.RunBuildPodController() + openshiftConfig.RunBuildImageChangeTriggerController() + if err := openshiftConfig.RunDeploymentController(); err != nil { + return err + } + openshiftConfig.RunDeployerPodController() + openshiftConfig.RunDeploymentConfigController() + openshiftConfig.RunDeploymentConfigChangeController() + openshiftConfig.RunDeploymentImageChangeTriggerController() + openshiftConfig.RunProjectAuthorizationCache() + + return nil +} diff --git a/pkg/cmd/server/start/start_node.go b/pkg/cmd/server/start/start_node.go new file mode 100644 index 000000000000..a7e071974f49 --- /dev/null +++ b/pkg/cmd/server/start/start_node.go @@ -0,0 +1,267 @@ +package start + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + _ "net/http/pprof" + "path" + "path/filepath" + "strings" + + "github.com/coreos/go-systemd/daemon" + "github.com/golang/glog" + "github.com/spf13/cobra" + + kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/record" + "github.com/openshift/origin/pkg/cmd/server/kubernetes" + + configapi "github.com/openshift/origin/pkg/cmd/server/api" + configapilatest "github.com/openshift/origin/pkg/cmd/server/api/latest" + "github.com/openshift/origin/pkg/cmd/server/certs" + cmdutil "github.com/openshift/origin/pkg/cmd/util" + "github.com/openshift/origin/pkg/cmd/util/docker" +) + +type NodeOptions struct { + NodeArgs *NodeArgs + + WriteConfigOnly bool + ConfigFile string +} + +const longNodeCommandDesc = ` +Start an OpenShift node +This command helps you launch an OpenShift node. Running + + $ openshift start node --master= + +will start an OpenShift node that attempts to connect to the master on the provided IP. The +node will run in the foreground until you terminate the process. +` + +// NewCommandStartMaster provides a CLI handler for 'start' command +func NewCommandStartNode() (*cobra.Command, *NodeOptions) { + options := &NodeOptions{} + + cmd := &cobra.Command{ + Use: "node", + Short: "Launch OpenShift node", + Long: longNodeCommandDesc, + Run: func(c *cobra.Command, args []string) { + if err := options.Complete(); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + if err := options.Validate(args); err != nil { + fmt.Println(err.Error()) + c.Help() + return + } + + if err := options.StartNode(); err != nil { + glog.Fatal(err) + } + }, + } + + flags := cmd.Flags() + + flags.BoolVar(&options.WriteConfigOnly, "write-config", false, "Indicates that the command should build the configuration from command-line arguments, write it to the location specified by --config, and exit.") + flags.StringVar(&options.ConfigFile, "config", "", "Location of the node configuration file to run from, or write to (when used with --write-config). When running from a configuration file, all other command-line arguments are ignored.") + + options.NodeArgs = NewDefaultNodeArgs() + // make sure that KubeConnectionArgs and NodeArgs use the same CertArgs for this command + options.NodeArgs.KubeConnectionArgs.CertArgs = options.NodeArgs.CertArgs + + BindNodeArgs(options.NodeArgs, flags, "") + BindBindAddrArg(options.NodeArgs.BindAddrArg, flags, "") + BindImageFormatArgs(options.NodeArgs.ImageFormatArgs, flags, "") + BindKubeConnectionArgs(options.NodeArgs.KubeConnectionArgs, flags, "") + BindCertArgs(options.NodeArgs.CertArgs, flags, "") + + return cmd, options +} + +func (o NodeOptions) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported for start node") + } + if o.WriteConfigOnly { + if len(o.ConfigFile) == 0 { + return errors.New("--config is required if --write-config is true") + } + } + + return nil +} + +func (o NodeOptions) Complete() error { + o.NodeArgs.NodeName = strings.ToLower(o.NodeArgs.NodeName) + + return nil +} + +// StartNode calls RunNode and then waits forever +func (o NodeOptions) StartNode() error { + if err := o.RunNode(); err != nil { + return err + } + + if o.WriteConfigOnly { + return nil + } + + daemon.SdNotify("READY=1") + select {} + + return nil +} + +// RunNode takes the options and: +// 1. Creates certs if needed +// 2. Reads fully specified node config OR builds a fully specified node config from the args +// 3. Writes the fully specified node config and exits if needed +// 4. Starts the node based on the fully specified config +func (o NodeOptions) RunNode() error { + startUsingConfigFile := !o.WriteConfigOnly && (len(o.ConfigFile) > 0) + mintCerts := o.NodeArgs.CertArgs.CreateCerts && !startUsingConfigFile + + if mintCerts { + if err := o.CreateCerts(); err != nil { + return nil + } + } + + var nodeConfig *configapi.NodeConfig + var err error + if startUsingConfigFile { + nodeConfig, err = ReadNodeConfig(o.ConfigFile) + } else { + nodeConfig, err = o.NodeArgs.BuildSerializeableNodeConfig() + } + if err != nil { + return err + } + + if o.WriteConfigOnly { + content, err := WriteNode(nodeConfig) + if err != nil { + return err + } + if err := ioutil.WriteFile(o.ConfigFile, content, 0644); err != nil { + return err + } + return nil + } + + _, kubeClientConfig, err := configapi.GetKubeClient(nodeConfig.MasterKubeConfig) + if err != nil { + return err + } + glog.Infof("Starting an OpenShift node, connecting to %s", kubeClientConfig.Host) + + if cmdutil.Env("OPENSHIFT_PROFILE", "") == "web" { + go func() { + glog.Infof("Starting profiling endpoint at http://127.0.0.1:6060/debug/pprof/") + glog.Fatal(http.ListenAndServe("127.0.0.1:6060", nil)) + }() + } + + if err := StartNode(*nodeConfig); err != nil { + return err + } + + return nil +} + +func (o NodeOptions) CreateCerts() error { + username := "node-" + o.NodeArgs.NodeName + signerOptions := &certs.CreateSignerCertOptions{ + CertFile: certs.DefaultCertFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + KeyFile: certs.DefaultKeyFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + SerialFile: certs.DefaultSerialFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + Name: certs.DefaultSignerName(), + } + if _, err := signerOptions.CreateSignerCert(); err != nil { + return err + } + getSignerOptions := &certs.GetSignerCertOptions{ + CertFile: certs.DefaultCertFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + KeyFile: certs.DefaultKeyFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + SerialFile: certs.DefaultSerialFilename(o.NodeArgs.CertArgs.CertDir, "ca"), + } + + mintNodeClientCert := certs.CreateNodeClientCertOptions{ + GetSignerCertOptions: getSignerOptions, + CertFile: certs.DefaultCertFilename(o.NodeArgs.CertArgs.CertDir, username), + KeyFile: certs.DefaultKeyFilename(o.NodeArgs.CertArgs.CertDir, username), + NodeName: o.NodeArgs.NodeName, + } + if _, err := mintNodeClientCert.CreateNodeClientCert(); err != nil { + return err + } + + masterAddr, err := o.NodeArgs.KubeConnectionArgs.GetKubernetesAddress(&o.NodeArgs.DefaultKubernetesURL) + if err != nil { + return err + } + + createKubeConfigOptions := certs.CreateKubeConfigOptions{ + APIServerURL: masterAddr.String(), + APIServerCAFile: getSignerOptions.CertFile, + ServerNick: "master", + + CertFile: mintNodeClientCert.CertFile, + KeyFile: mintNodeClientCert.KeyFile, + UserNick: username, + + KubeConfigFile: path.Join(filepath.Dir(mintNodeClientCert.CertFile), ".kubeconfig"), + } + if _, err := createKubeConfigOptions.CreateKubeConfig(); err != nil { + return err + } + + return nil +} + +func ReadNodeConfig(filename string) (*configapi.NodeConfig, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + config := &configapi.NodeConfig{} + + if err := configapilatest.Codec.DecodeInto(data, config); err != nil { + return nil, err + } + return config, nil +} + +func StartNode(config configapi.NodeConfig) error { + if config.RecordEvents { + kubeClient, _, err := configapi.GetKubeClient(config.MasterKubeConfig) + if err != nil { + return err + } + + // TODO: recording should occur in individual components + record.StartRecording(kubeClient.Events(""), kapi.EventSource{Component: "node"}) + } + + nodeConfig, err := kubernetes.BuildKubernetesNodeConfig(config) + if err != nil { + return err + } + + nodeConfig.EnsureVolumeDir() + nodeConfig.EnsureDocker(docker.NewHelper()) + nodeConfig.RunProxy() + nodeConfig.RunKubelet() + + return nil +} diff --git a/pkg/cmd/util/env.go b/pkg/cmd/util/env.go index 3dca6f18604c..b43f15a1fb2b 100644 --- a/pkg/cmd/util/env.go +++ b/pkg/cmd/util/env.go @@ -4,8 +4,17 @@ import ( "fmt" "os" "regexp" + "strconv" ) +func EnvInt(key string, defaultValue int32, minValue int32) int32 { + value, err := strconv.ParseInt(Env(key, fmt.Sprintf("%d", defaultValue)), 10, 32) + if err != nil || int32(value) < minValue { + return defaultValue + } + return int32(value) +} + // Env returns an environment variable or a default value if not specified. func Env(key string, defaultValue string) string { val := os.Getenv(key) diff --git a/pkg/cmd/util/net.go b/pkg/cmd/util/net.go index cfa53f4ec1ed..5c1b1384126a 100644 --- a/pkg/cmd/util/net.go +++ b/pkg/cmd/util/net.go @@ -9,14 +9,14 @@ import ( ) // TryListen tries to open a connection on the given port and returns true if it succeeded. -func TryListen(hostPort string) bool { +func TryListen(hostPort string) (bool, error) { l, err := net.Listen("tcp", hostPort) if err != nil { glog.V(5).Infof("Failure while checking listen on %s: %v", err) - return false + return false, err } defer l.Close() - return true + return true, nil } // WaitForDial attempts to connect to the given address, closing and returning nil on the first successful connection. diff --git a/test/integration/authorization_test.go b/test/integration/authorization_test.go index cd533a5de855..84cf29c1ba42 100644 --- a/test/integration/authorization_test.go +++ b/test/integration/authorization_test.go @@ -17,21 +17,21 @@ import ( ) func TestRestrictedAccessForProjectAdmins(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - openshiftClient, openshiftClientConfig, err := startConfig.GetOpenshiftClient() + clusterAdminClient, _, clusterAdminClientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Errorf("unexpected error: %v", err) } - haroldClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "hammer-project", "harold") + haroldClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "hammer-project", "harold") if err != nil { t.Errorf("unexpected error: %v", err) } - markClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "mallet-project", "mark") + markClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "mallet-project", "mark") if err != nil { t.Errorf("unexpected error: %v", err) } @@ -81,12 +81,12 @@ func TestRestrictedAccessForProjectAdmins(t *testing.T) { } func TestOnlyResolveRolesForBindingsThatMatter(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - openshiftClient, _, err := startConfig.GetOpenshiftClient() + clusterAdminClient, _, _, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -95,14 +95,14 @@ func TestOnlyResolveRolesForBindingsThatMatter(t *testing.T) { RoleNamespace: "master", RoleName: "view", BindingNamespace: "master", - Client: openshiftClient, + Client: clusterAdminClient, Users: []string{"anypassword:valerie"}, } if err := addValerie.Run(); err != nil { t.Errorf("unexpected error: %v", err) } - if err = openshiftClient.Roles("master").Delete("view"); err != nil { + if err = clusterAdminClient.Roles("master").Delete("view"); err != nil { t.Errorf("unexpected error: %v", err) } @@ -110,7 +110,7 @@ func TestOnlyResolveRolesForBindingsThatMatter(t *testing.T) { RoleNamespace: "master", RoleName: "edit", BindingNamespace: "master", - Client: openshiftClient, + Client: clusterAdminClient, Users: []string{"anypassword:edgar"}, } if err := addEdgar.Run(); err != nil { @@ -126,7 +126,7 @@ func TestOnlyResolveRolesForBindingsThatMatter(t *testing.T) { // TODO this list should start collapsing as we continue to tighten access on generated system ids var globalClusterAdminUsers = util.NewStringSet("system:kube-client", "system:openshift-client", "system:openshift-deployer") -var globalClusterAdminGroups = util.NewStringSet("system:cluster-admins") +var globalClusterAdminGroups = util.NewStringSet("system:cluster-admins", "system:nodes") type resourceAccessReviewTest struct { clientInterface client.ResourceAccessReviewInterface @@ -156,21 +156,21 @@ func (test resourceAccessReviewTest) run(t *testing.T) { } func TestResourceAccessReview(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - openshiftClient, openshiftClientConfig, err := startConfig.GetOpenshiftClient() + clusterAdminClient, _, clusterAdminClientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Errorf("unexpected error: %v", err) } - haroldClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "hammer-project", "harold") + haroldClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "hammer-project", "harold") if err != nil { t.Errorf("unexpected error: %v", err) } - markClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "mallet-project", "mark") + markClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "mallet-project", "mark") if err != nil { t.Errorf("unexpected error: %v", err) } @@ -239,7 +239,7 @@ func TestResourceAccessReview(t *testing.T) { // a cluster-admin should be able to make global access review requests { test := resourceAccessReviewTest{ - clientInterface: openshiftClient.RootResourceAccessReviews(), + clientInterface: clusterAdminClient.RootResourceAccessReviews(), review: requestWhoCanViewDeployments, response: authorizationapi.ResourceAccessReviewResponse{ Users: globalClusterAdminUsers, @@ -278,21 +278,21 @@ func (test subjectAccessReviewTest) run(t *testing.T) { } func TestSubjectAccessReview(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - openshiftClient, openshiftClientConfig, err := startConfig.GetOpenshiftClient() + clusterAdminClient, _, clusterAdminClientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Errorf("unexpected error: %v", err) } - haroldClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "hammer-project", "harold") + haroldClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "hammer-project", "harold") if err != nil { t.Errorf("unexpected error: %v", err) } - markClient, err := CreateNewProject(openshiftClient, *openshiftClientConfig, "mallet-project", "mark") + markClient, err := CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, "mallet-project", "mark") if err != nil { t.Errorf("unexpected error: %v", err) } @@ -368,7 +368,7 @@ func TestSubjectAccessReview(t *testing.T) { askCanClusterAdminsCreateProject := &authorizationapi.SubjectAccessReview{Groups: util.NewStringSet("system:cluster-admins"), Verb: "create", Resource: "projects"} subjectAccessReviewTest{ - clientInterface: openshiftClient.RootSubjectAccessReviews(), + clientInterface: clusterAdminClient.RootSubjectAccessReviews(), review: askCanClusterAdminsCreateProject, response: authorizationapi.SubjectAccessReviewResponse{ Allowed: true, diff --git a/test/integration/bootstrap_policy_test.go b/test/integration/bootstrap_policy_test.go index 2d5514128954..b0380bf00120 100644 --- a/test/integration/bootstrap_policy_test.go +++ b/test/integration/bootstrap_policy_test.go @@ -15,17 +15,17 @@ import ( ) func TestAuthenticatedUsersAgainstOpenshiftNamespace(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - _, openshiftClientConfig, err := startConfig.GetOpenshiftClient() + _, _, clusterAdminClientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Errorf("unexpected error: %v", err) } - valerieClientConfig := *openshiftClientConfig + valerieClientConfig := *clusterAdminClientConfig valerieClientConfig.Username = "" valerieClientConfig.Password = "" valerieClientConfig.BearerToken = "" diff --git a/test/integration/dns_test.go b/test/integration/dns_test.go index 76b33d557a52..a9d47593fb39 100644 --- a/test/integration/dns_test.go +++ b/test/integration/dns_test.go @@ -12,14 +12,11 @@ import ( ) func TestDNS(t *testing.T) { - _, err := StartTestAllInOne() + masterConfig, _, err := StartTestAllInOne() if err != nil { t.Fatalf("unexpected error: %v", err) } - // ugly... - server := "127.0.0.1:8053" - // verify service DNS entry is visible stop := make(chan struct{}) util.Until(func() { @@ -27,7 +24,7 @@ func TestDNS(t *testing.T) { MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: true}, Question: []dns.Question{{"kubernetes.default.local.", dns.TypeA, dns.ClassINET}}, } - in, err := dns.Exchange(m1, server) + in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress) if err != nil { t.Logf("unexpected error: %v", err) return @@ -52,7 +49,7 @@ func TestDNS(t *testing.T) { MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: true}, Question: []dns.Question{{"foo.kubernetes.default.local.", dns.TypeA, dns.ClassINET}}, } - in, err := dns.Exchange(m1, server) + in, err := dns.Exchange(m1, masterConfig.DNSConfig.BindAddress) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -73,7 +70,7 @@ func TestDNS(t *testing.T) { MsgHdr: dns.MsgHdr{Id: dns.Id(), RecursionDesired: true}, Question: []dns.Question{{"www.google.com.", dns.TypeA, dns.ClassINET}}, } - in, err = dns.Exchange(m1, server) + in, err = dns.Exchange(m1, masterConfig.DNSConfig.BindAddress) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/test/integration/login_test.go b/test/integration/login_test.go index 03404b89eb79..7f529c9f44a4 100644 --- a/test/integration/login_test.go +++ b/test/integration/login_test.go @@ -20,12 +20,12 @@ func init() { } func TestLogin(t *testing.T) { - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - openshiftClient, openshiftClientConfig, err := startConfig.GetOpenshiftClient() + clusterAdminClient, _, clusterAdminClientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -40,7 +40,7 @@ func TestLogin(t *testing.T) { username := "joe" password := "pass" project := "the-singularity-is-near" - server := openshiftClientConfig.Host + server := clusterAdminClientConfig.Host loginOptions = newLoginOptions(server, username, password, "", true) @@ -61,7 +61,7 @@ func TestLogin(t *testing.T) { } newProjectOptions := &newproject.NewProjectOptions{ - Client: openshiftClient, + Client: clusterAdminClient, ProjectName: project, AdminRole: "admin", MasterPolicyNamespace: "master", diff --git a/test/integration/oauth_htpasswd_test.go b/test/integration/oauth_htpasswd_test.go index 1750f316f926..76ac006f4ea3 100644 --- a/test/integration/oauth_htpasswd_test.go +++ b/test/integration/oauth_htpasswd_test.go @@ -25,13 +25,14 @@ func TestHTPasswd(t *testing.T) { os.Setenv("OPENSHIFT_OAUTH_PASSWORD_AUTH", "htpasswd") os.Setenv("OPENSHIFT_OAUTH_HTPASSWD_FILE", htpasswdFile.Name()) - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestMaster() if err != nil { t.Fatalf("unexpected error: %v", err) } - _, clientConfig, err := startConfig.GetOpenshiftClient() + + _, _, clientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { - t.Fatalf("unexpected error: %v", err) + t.Errorf("unexpected error: %v", err) } // Use the server and CA info diff --git a/test/integration/oauth_request_header_test.go b/test/integration/oauth_request_header_test.go index 74f45a8e4def..774ba7802141 100644 --- a/test/integration/oauth_request_header_test.go +++ b/test/integration/oauth_request_header_test.go @@ -104,11 +104,12 @@ func TestOAuthRequestHeader(t *testing.T) { os.Setenv("OPENSHIFT_OAUTH_REQUEST_HEADER_CA_FILE", caFile.Name()) // Start server - startConfig, err := StartTestMaster() + _, clusterAdminKubeConfig, err := StartTestAllInOne() if err != nil { t.Fatalf("unexpected error: %v", err) } - _, clientConfig, err := startConfig.GetOpenshiftClient() + + _, _, clientConfig, err := GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/test/integration/server_test.go b/test/integration/server_test.go index ebe4116b8596..e0e7233302a3 100644 --- a/test/integration/server_test.go +++ b/test/integration/server_test.go @@ -11,10 +11,12 @@ import ( kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/openshift/origin/pkg/client" newproject "github.com/openshift/origin/pkg/cmd/experimental/project" - start "github.com/openshift/origin/pkg/cmd/server" + configapi "github.com/openshift/origin/pkg/cmd/server/api" + "github.com/openshift/origin/pkg/cmd/server/start" cmdutil "github.com/openshift/origin/pkg/cmd/util" "github.com/openshift/origin/pkg/cmd/util/tokencmd" ) @@ -23,53 +25,71 @@ func init() { requireEtcd() } -func StartTestServer(args ...string) (start.Config, error) { - deleteAllEtcdKeys() - - startConfig := start.NewDefaultConfig() - startConfig.DNSBindAddr.DefaultPort = 8053 - startConfig.DNSBindAddr = startConfig.DNSBindAddr.Default() +func setupStartOptions() (*start.MasterArgs, *start.NodeArgs, *start.BindAddrArg, *start.ImageFormatArgs, *start.KubeConnectionArgs, *start.CertArgs) { + masterArgs, nodeArgs, bindAddrArg, imageFormatArgs, kubeConnectionArgs, certArgs := start.GetAllInOneArgs() basedir := path.Join(os.TempDir(), "openshift-integration-tests") - - startConfig.VolumeDir = path.Join(basedir, "volume") - startConfig.EtcdDir = path.Join(basedir, "etcd") - startConfig.CertDir = path.Join(basedir, "cert") + nodeArgs.VolumeDir = path.Join(basedir, "volume") + masterArgs.EtcdDir = path.Join(basedir, "etcd") + certArgs.CertDir = path.Join(basedir, "cert") // don't wait for nodes to come up - if len(args) > 0 && args[0] == "master" { - startConfig.NodeList = nil - } masterAddr := httptest.NewUnstartedServer(nil).Listener.Addr().String() fmt.Printf("masterAddr: %#v\n", masterAddr) - startConfig.MasterAddr.Set(masterAddr) - startConfig.BindAddr.Set(masterAddr) - startConfig.EtcdAddr.Set(getEtcdURL()) + masterArgs.MasterAddr.Set(masterAddr) + bindAddrArg.BindAddr.Set(masterAddr) + masterArgs.EtcdAddr.Set(getEtcdURL()) assetAddr := httptest.NewUnstartedServer(nil).Listener.Addr().String() fmt.Printf("assetAddr: %#v\n", assetAddr) - startConfig.AssetBindAddr.Set(assetAddr) - startConfig.AssetPublicAddr.Set(assetAddr) + masterArgs.AssetBindAddr.Set(assetAddr) + masterArgs.AssetPublicAddr.Set(assetAddr) + + dnsAddr := httptest.NewUnstartedServer(nil).Listener.Addr().String() + fmt.Printf("dnsAddr: %#v\n", dnsAddr) + masterArgs.DNSBindAddr.Set(dnsAddr) + + return masterArgs, nodeArgs, bindAddrArg, imageFormatArgs, kubeConnectionArgs, certArgs +} + +func getAdminKubeConfigFile(certArgs start.CertArgs) string { + return path.Clean(path.Join(certArgs.CertDir, "admin/.kubeconfig")) +} + +func StartTestAllInOne() (*configapi.MasterConfig, string, error) { + deleteAllEtcdKeys() + + masterArgs, nodeArgs, _, _, _, _ := setupStartOptions() + masterArgs.NodeList = nil - startConfig.Complete(args) + startOptions := start.AllInOneOptions{} + startOptions.MasterArgs, startOptions.NodeArgs = masterArgs, nodeArgs + startOptions.Complete() errCh := make(chan error) go func() { - errCh <- startConfig.Start(args) + errCh <- startOptions.StartAllInOne() close(errCh) }() + adminKubeConfigFile := getAdminKubeConfigFile(*masterArgs.CertArgs) + + openshiftMasterConfig, err := startOptions.MasterArgs.BuildSerializeableMasterConfig() + if err != nil { + return nil, "", err + } + // wait for the server to come up: 35 seconds - if err := cmdutil.WaitForSuccessfulDial(true, "tcp", masterAddr, 100*time.Millisecond, 1*time.Second, 35); err != nil { + if err := cmdutil.WaitForSuccessfulDial(true, "tcp", masterArgs.MasterAddr.URL.Host, 100*time.Millisecond, 1*time.Second, 35); err != nil { select { case err := <-errCh: if err != nil { - return *startConfig, err + return nil, "", err } default: } - return *startConfig, err + return nil, "", err } // try to get a client @@ -77,34 +97,74 @@ func StartTestServer(args ...string) (start.Config, error) { select { case err := <-errCh: if err != nil { - return *startConfig, err + return nil, "", err } default: } // confirm that we can actually query from the api server - if client, _, err := startConfig.GetOpenshiftClient(); err == nil { + + if client, _, _, err := GetClusterAdminClient(adminKubeConfigFile); err == nil { if _, err := client.Policies("master").List(labels.Everything(), labels.Everything()); err == nil { break } } time.Sleep(100 * time.Millisecond) } - return *startConfig, nil + return openshiftMasterConfig, adminKubeConfigFile, nil } -// StartTestMaster starts up a test master and returns back the startConfig so you can get clients and certs -func StartTestMaster() (start.Config, error) { - return StartTestServer("master") -} +// TODO Unify with StartAllInOne. -// StartTestNode starts up a test node and returns back the startConfig so you can get clients and certs -func StartTestNode() (start.Config, error) { - return StartTestServer("node") -} +// StartTestMaster starts up a test master and returns back the startOptions so you can get clients and certs +func StartTestMaster() (*configapi.MasterConfig, string, error) { + deleteAllEtcdKeys() + + masterArgs, _, _, _, _, _ := setupStartOptions() + + startOptions := start.MasterOptions{} + startOptions.MasterArgs = masterArgs + startOptions.Complete() + + var startError error + go func() { + err := startOptions.StartMaster() + if err != nil { + startError = err + fmt.Printf("ERROR STARTING SERVER! %v", err) + } + }() + + adminKubeConfigFile := getAdminKubeConfigFile(*masterArgs.CertArgs) + + openshiftMasterConfig, err := startOptions.MasterArgs.BuildSerializeableMasterConfig() + if err != nil { + return nil, "", err + } + + // wait for the server to come up: 35 seconds + if err := cmdutil.WaitForSuccessfulDial(true, "tcp", masterArgs.MasterAddr.URL.Host, 100*time.Millisecond, 1*time.Second, 35); err != nil { + return nil, "", err + } + + stopChannel := make(chan struct{}) + util.Until( + func() { + if startError != nil { + close(stopChannel) + return + } + + // confirm that we can actually query from the api server + client, _, _, err := GetClusterAdminClient(adminKubeConfigFile) + if err != nil { + return + } + if _, err := client.Policies("master").List(labels.Everything(), labels.Everything()); err == nil { + close(stopChannel) + } + }, 100*time.Millisecond, stopChannel) -// StartTestAllInOne starts up a test all-in-one and returns back the startConfig so you can get clients and certs -func StartTestAllInOne() (start.Config, error) { - return StartTestServer() + return openshiftMasterConfig, adminKubeConfigFile, startError } // CreateNewProject creates a new project using the clusterAdminClient, then gets a token for the adminUser and returns @@ -144,3 +204,17 @@ func CreateNewProject(clusterAdminClient *client.Client, clientConfig kclient.Co return adminClient, nil } + +func GetClusterAdminClient(adminKubeConfigFile string) (*client.Client, *kclient.Client, *kclient.Config, error) { + kclient, clientConfig, err := configapi.GetKubeClient(adminKubeConfigFile) + if err != nil { + return nil, nil, nil, err + } + + osclient, err := client.New(clientConfig) + if err != nil { + return nil, nil, nil, err + } + + return osclient, kclient, clientConfig, nil +}