Skip to content

Introduce Agent platform #760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-robot merged 2 commits into from
Dec 14, 2021
Merged

Introduce Agent platform #760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3aaa355
Introduce Agent platform
avishayt Dec 2, 2021
95604cc
Share code between None and Agent platforms
avishayt Dec 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 50 additions & 36 deletions api/fixtures/example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ type ExampleOptions struct {
EtcdStorageClass string
AWS *ExampleAWSOptions
None *ExampleNoneOptions
Agent *ExampleAgentOptions
NetworkType hyperv1.NetworkType
ControlPlaneAvailabilityPolicy hyperv1.AvailabilityPolicy
InfrastructureAvailabilityPolicy hyperv1.AvailabilityPolicy
Expand All @@ -73,6 +74,10 @@ type ExampleNoneOptions struct {
APIServerAddress string
}

type ExampleAgentOptions struct {
APIServerAddress string
}

type ExampleAWSOptions struct {
Region string
Zone string
Expand Down Expand Up @@ -223,43 +228,12 @@ web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token
platformSpec = hyperv1.PlatformSpec{
Type: hyperv1.NonePlatform,
}
services = []hyperv1.ServicePublishingStrategyMapping{
{
Service: hyperv1.APIServer,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: o.None.APIServerAddress},
},
},
{
Service: hyperv1.OAuthServer,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: o.None.APIServerAddress},
},
},
{
Service: hyperv1.OIDC,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.None,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: o.None.APIServerAddress},
},
},
{
Service: hyperv1.Konnectivity,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: o.None.APIServerAddress},
},
},
{
Service: hyperv1.Ignition,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: o.None.APIServerAddress},
},
},
services = o.getServicePublishingStrategyMappingByAPIServerAddress(o.None.APIServerAddress)
case o.Agent != nil:
platformSpec = hyperv1.PlatformSpec{
Type: hyperv1.AgentPlatform,
}
services = o.getServicePublishingStrategyMappingByAPIServerAddress(o.Agent.APIServerAddress)

default:
panic("no platform specified")
Expand Down Expand Up @@ -376,3 +350,43 @@ web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token
NodePool: nodePool,
}
}

func (o ExampleOptions) getServicePublishingStrategyMappingByAPIServerAddress(APIServerAddress string) []hyperv1.ServicePublishingStrategyMapping {
return []hyperv1.ServicePublishingStrategyMapping{
{
Service: hyperv1.APIServer,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: APIServerAddress},
},
},
{
Service: hyperv1.OAuthServer,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: APIServerAddress},
},
},
{
Service: hyperv1.OIDC,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.None,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: APIServerAddress},
},
},
{
Service: hyperv1.Konnectivity,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: APIServerAddress},
},
},
{
Service: hyperv1.Ignition,
ServicePublishingStrategy: hyperv1.ServicePublishingStrategy{
Type: hyperv1.NodePort,
NodePort: &hyperv1.NodePortPublishingStrategy{Address: APIServerAddress},
},
},
}
}
2 changes: 2 additions & 0 deletions api/scheme.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
operatorv1 "github.com/openshift/api/operator/v1"
routev1 "github.com/openshift/api/route/v1"
securityv1 "github.com/openshift/api/security/v1"
agentv1 "github.com/openshift/cluster-api-provider-agent/api/v1alpha1"
hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -48,4 +49,5 @@ func init() {
apiextensionsv1.AddToScheme(Scheme)
kasv1beta1.AddToScheme(Scheme)
prometheusoperatorv1.AddToScheme(Scheme)
agentv1.AddToScheme(Scheme)
}
5 changes: 4 additions & 1 deletion api/v1alpha1/hostedcluster_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ const (

// PlatformType is a specific supported infrastructure provider.
//
// +kubebuilder:validation:Enum=AWS;None;IBMCloud
// +kubebuilder:validation:Enum=AWS;None;IBMCloud;Agent
type PlatformType string

const (
Expand All @@ -396,6 +396,9 @@ const (

// IBMCloudPlatform represents IBM Cloud infrastructure.
IBMCloudPlatform PlatformType = "IBMCloud"

// AgentPlatform represents user supplied insfrastructure booted with agents.
AgentPlatform PlatformType = "Agent"
)

// PlatformSpec specifies the underlying infrastructure provider for the cluster
Expand Down
70 changes: 70 additions & 0 deletions cmd/cluster/agent/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package agent

import (
"context"
"fmt"
"os"
"os/signal"
"syscall"

apifixtures "github.com/openshift/hypershift/api/fixtures"
"github.com/openshift/hypershift/cmd/cluster/core"
"github.com/spf13/cobra"
utilrand "k8s.io/apimachinery/pkg/util/rand"
)

func NewCreateCommand(opts *core.CreateOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "agent",
Short: "Creates basic functional HostedCluster resources on Agent",
SilenceUsage: true,
}

opts.AgentPlatform = core.AgentPlatformCreateOptions{}

cmd.Run = func(cmd *cobra.Command, args []string) {
ctx, cancel := context.WithCancel(context.Background())
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT)
go func() {
<-sigs
cancel()
}()

if err := CreateCluster(ctx, opts); err != nil {
log.Error(err, "Failed to create cluster")
os.Exit(1)
}
}

return cmd
}

func CreateCluster(ctx context.Context, opts *core.CreateOptions) error {
return core.CreateCluster(ctx, opts, applyPlatformSpecificsValues)
}

func applyPlatformSpecificsValues(ctx context.Context, exampleOptions *apifixtures.ExampleOptions, opts *core.CreateOptions) (err error) {
if opts.AgentPlatform.APIServerAddress == "" {
opts.AgentPlatform.APIServerAddress, err = core.GetAPIServerAddressByNode(ctx)
if err != nil {
return err
}
}

infraID := opts.InfraID
if len(infraID) == 0 {
infraID = fmt.Sprintf("%s-%s", opts.Name, utilrand.String(5))
}
exampleOptions.InfraID = infraID

// TODO: We are inconsistent atm as DNS basedomain input is required and platform agnostic at the HostedCluster API level.
// However In the CLI input is only exposed in AWS and not required atm. This should either be core input and required for
// every platform or the field in the API should be optional if there's platform which don't need it.
exampleOptions.BaseDomain = "example.com"

exampleOptions.Agent = &apifixtures.ExampleAgentOptions{
APIServerAddress: opts.AgentPlatform.APIServerAddress,
}
return nil
}
48 changes: 48 additions & 0 deletions cmd/cluster/agent/destroy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package agent

import (
"context"
"os"
"os/signal"
"syscall"
"time"

"github.com/openshift/hypershift/cmd/cluster/core"
"github.com/openshift/hypershift/cmd/cluster/none"
"github.com/spf13/cobra"
)

type DestroyOptions struct {
Namespace string
Name string
ClusterGracePeriod time.Duration
}

func NewDestroyCommand(opts *core.DestroyOptions) *cobra.Command {
cmd := &cobra.Command{
Use: "agent",
Short: "Destroys a HostedCluster and its associated infrastructure on Agent.",
SilenceUsage: true,
}

cmd.Run = func(cmd *cobra.Command, args []string) {
ctx, cancel := context.WithCancel(context.Background())
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT)
go func() {
<-sigs
cancel()
}()

if err := DestroyCluster(ctx, opts); err != nil {
log.Error(err, "Failed to destroy cluster")
os.Exit(1)
}
}

return cmd
}

func DestroyCluster(ctx context.Context, o *core.DestroyOptions) error {
return none.DestroyCluster(ctx, o)
}
7 changes: 7 additions & 0 deletions cmd/cluster/agent/log.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package agent

import (
"github.com/openshift/hypershift/cmd/util"
)

var log = util.Log
3 changes: 3 additions & 0 deletions cmd/cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"time"

hyperv1 "github.com/openshift/hypershift/api/v1alpha1"
"github.com/openshift/hypershift/cmd/cluster/agent"
"github.com/openshift/hypershift/cmd/cluster/aws"
"github.com/openshift/hypershift/cmd/cluster/core"
"github.com/openshift/hypershift/cmd/cluster/none"
Expand Down Expand Up @@ -50,6 +51,7 @@ func NewCreateCommands() *cobra.Command {

cmd.AddCommand(aws.NewCreateCommand(opts))
cmd.AddCommand(none.NewCreateCommand(opts))
cmd.AddCommand(agent.NewCreateCommand(opts))

return cmd
}
Expand All @@ -76,6 +78,7 @@ func NewDestroyCommands() *cobra.Command {

cmd.AddCommand(aws.NewDestroyCommand(opts))
cmd.AddCommand(none.NewDestroyCommand(opts))
cmd.AddCommand(agent.NewDestroyCommand(opts))

return cmd
}
41 changes: 41 additions & 0 deletions cmd/cluster/core/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ import (
"github.com/openshift/hypershift/cmd/version"
hyperapi "github.com/openshift/hypershift/support/api"
"golang.org/x/crypto/ssh"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeclient "k8s.io/client-go/kubernetes"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
)

Expand Down Expand Up @@ -44,6 +47,11 @@ type CreateOptions struct {
SSHKeyFile string
NonePlatform NonePlatformCreateOptions
AWSPlatform AWSPlatformOptions
AgentPlatform AgentPlatformCreateOptions
}

type AgentPlatformCreateOptions struct {
APIServerAddress string
}

type NonePlatformCreateOptions struct {
Expand Down Expand Up @@ -178,6 +186,39 @@ func apply(ctx context.Context, exampleOptions *apifixtures.ExampleOptions, rend
return nil
}

func GetAPIServerAddressByNode(ctx context.Context) (string, error) {
// Fetch a single node and determine possible DNS or IP entries to use
// for external node-port communication.
// Possible values are considered with the following priority based on the address type:
// - NodeExternalDNS
// - NodeExternalIP
// - NodeInternalIP
apiServerAddress := ""
kubeClient := kubeclient.NewForConfigOrDie(util.GetConfigOrDie())
nodes, err := kubeClient.CoreV1().Nodes().List(ctx, v1.ListOptions{Limit: 1})
if err != nil {
return "", fmt.Errorf("unable to fetch node objects: %w", err)
}
if len(nodes.Items) < 1 {
return "", fmt.Errorf("no node objects found: %w", err)
Copy link
Contributor

@alvaroaleman alvaroaleman Dec 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The err here is always nil, as you return before if its non-nil

Copy link
Contributor Author

@avishayt avishayt Dec 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code was moved from the None platform as-is for reuse by the Agent platform, I'd rather not change it in this PR

Copy link
Member

@enxebre enxebre Dec 13, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The err here is always nil, as you return before if its non-nil

Why though? would .List though an error if nodes.Items is of len 0? If so +1 to address separately and keep this PR focus on the multi-platform structure.

}
addresses := map[corev1.NodeAddressType]string{}
for _, address := range nodes.Items[0].Status.Addresses {
addresses[address.Type] = address.Address
}
for _, addrType := range []corev1.NodeAddressType{corev1.NodeExternalDNS, corev1.NodeExternalIP, corev1.NodeInternalIP} {
if address, exists := addresses[addrType]; exists {
apiServerAddress = address
break
}
}
if apiServerAddress == "" {
return "", fmt.Errorf("node %q does not expose any IP addresses, this should not be possible", nodes.Items[0].Name)
}
log.Info(fmt.Sprintf("detected %q from node %q as external-api-server-address", apiServerAddress, nodes.Items[0].Name))
return apiServerAddress, nil
}

func CreateCluster(ctx context.Context, opts *CreateOptions, platformSpecificApply ApplyPlatformSpecifics) error {
exampleOptions, err := createCommonFixture(opts)
if err != nil {
Expand Down
Loading