diff --git a/_topic_map.yml b/_topic_map.yml index 46f27d671e54..f87aa9522d07 100644 --- a/_topic_map.yml +++ b/_topic_map.yml @@ -83,6 +83,11 @@ Topics: File: installing-aws-network-customizations - Name: Uninstalling a cluster on AWS File: uninstalling-cluster-aws +- Name: Installing on AWS UPI + Dir: installing_aws_upi + Topics: + - Name: Installing a cluster on AWS using CloudFormation templates + File: installing-aws-upi - Name: Installing on bare metal Dir: installing_bare_metal Topics: diff --git a/installing/installing_aws/installing-aws-customizations.adoc b/installing/installing_aws/installing-aws-customizations.adoc index ba0da9a953da..11165f40f5d2 100644 --- a/installing/installing_aws/installing-aws-customizations.adoc +++ b/installing/installing_aws/installing-aws-customizations.adoc @@ -15,8 +15,6 @@ to host the cluster. include::modules/installation-overview.adoc[leveloffset=+1] -include::modules/installation-clouds.adoc[leveloffset=+1] - include::modules/installation-provide-credentials.adoc[leveloffset=+1] include::modules/installation-obtaining-installer.adoc[leveloffset=+1] diff --git a/installing/installing_aws/installing-aws-default.adoc b/installing/installing_aws/installing-aws-default.adoc index fbfd2c7bd3a5..606eb83e57eb 100644 --- a/installing/installing_aws/installing-aws-default.adoc +++ b/installing/installing_aws/installing-aws-default.adoc @@ -15,8 +15,6 @@ to host the cluster. include::modules/installation-overview.adoc[leveloffset=+1] -include::modules/installation-clouds.adoc[leveloffset=+1] - include::modules/installation-provide-credentials.adoc[leveloffset=+1] include::modules/installation-obtaining-installer.adoc[leveloffset=+1] diff --git a/installing/installing_aws_upi/installing-aws-upi.adoc b/installing/installing_aws_upi/installing-aws-upi.adoc new file mode 100644 index 000000000000..55af783e0a5f --- /dev/null +++ b/installing/installing_aws_upi/installing-aws-upi.adoc @@ -0,0 +1,78 @@ +[id="installing-aws-upi"] += Installing a cluster on AWS using CloudFormation templates +include::modules/common-attributes.adoc[] +:context: installing-aws-upi + +toc::[] + +In {product-title} version {product-version}, you can install a +cluster on Amazon Web Services (AWS) using infrastructure that you provide. + +One way to create this infrastructure is to use the provided +CloudFormation templates. You can modify the templates to customize your +infrastructure or use the information that they contain to create AWS objects +according to your company's policies. + +.Prerequisites + +* xref:../../installing/installing_aws/installing-aws-account.adoc[Configure an AWS account] +to host the cluster. + +include::modules/installation-overview.adoc[leveloffset=+1] + +include::modules/installation-aws-upi-requirements.adoc[leveloffset=+1] + +include::modules/installation-aws-permissions.adoc[leveloffset=+2] + +include::modules/installation-obtaining-installer.adoc[leveloffset=+1] + +include::modules/installation-provide-credentials.adoc[leveloffset=+1] + +include::modules/installation-generate-aws-upi.adoc[leveloffset=+1] + +include::modules/installation-extracting-infraid.adoc[leveloffset=+1] + +include::modules/installation-creating-aws-vpc.adoc[leveloffset=+1] + +include::modules/installation-cloudformation-vpc.adoc[leveloffset=+2] + +include::modules/installation-creating-aws-dns.adoc[leveloffset=+1] + +include::modules/installation-cloudformation-dns.adoc[leveloffset=+2] + +include::modules/installation-creating-aws-security.adoc[leveloffset=+1] + +include::modules/installation-cloudformation-security.adoc[leveloffset=+2] + +include::modules/installation-aws-upi-rhcos-ami.adoc[leveloffset=+1] + +include::modules/installation-creating-aws-bootstrap.adoc[leveloffset=+1] + +include::modules/installation-cloudformation-bootstrap.adoc[leveloffset=+2] + +include::modules/installation-creating-aws-control-plane.adoc[leveloffset=+1] + +include::modules/installation-cloudformation-control-plane.adoc[leveloffset=+2] + +include::modules/installation-aws-upi-bootstrap.adoc[leveloffset=+1] + +//// +[id="installing-workers-aws-upi"] +== Creating worker nodes + +You can either manually create worker nodes or use a MachineSet to create worker +nodes after the cluster deploys. If you use a MachineSet to create and maintain +the workers, you can allow the cluster to manage them. This allows you to easily +scale, manage, and upgrade your workers. +//// + + +include::modules/installation-creating-aws-worker.adoc[leveloffset=+2] + +include::modules/installation-cloudformation-worker.adoc[leveloffset=+3] + +include::modules/cli-install.adoc[leveloffset=+1] + +include::modules/cli-logging-in-kubeadmin.adoc[leveloffset=+1] + +include::modules/installation-aws-upi-installation.adoc[leveloffset=+1] \ No newline at end of file diff --git a/modules/installation-aws-permissions.adoc b/modules/installation-aws-permissions.adoc index b3ce6bfe2faf..5f2441f27b6b 100644 --- a/modules/installation-aws-permissions.adoc +++ b/modules/installation-aws-permissions.adoc @@ -32,7 +32,7 @@ cluster, the IAM user requires the following permissions: |`bootstrap` |`s3:GetObject` |Yes -|Allows fetching Ignition configs from installation bucket? +|Allows fetching Ignition config files from the installation bucket |`master` |`elasticloadbalancing:*` diff --git a/modules/installation-aws-upi-bootstrap.adoc b/modules/installation-aws-upi-bootstrap.adoc new file mode 100644 index 000000000000..f52115d4bc4d --- /dev/null +++ b/modules/installation-aws-upi-bootstrap.adoc @@ -0,0 +1,34 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-aws-upi-bootstrap-{context}"] += Initializing the bootstrap node on AWS UPI + +After you create all of the required infrastructure in Amazon Web Services (AWS), +you can install the cluster. + +.Prerequisites + +* Configure an AWS account. +* Generate the Ignition config files for your cluster. +* Create and configure a VPC and assocated subnets in AWS. +* Create and configure DNS, load balancers, and listeners in AWS. +* Create control plane and compute roles. +* Create the bootstrap machine. +* Create the control plane machines. +* If you plan to manually manage the worker machines, create the worker machines. + +.Procedure + +. Change to the directory that contains the installation program and run the +following command: ++ +---- +$ ./openshift-install wait-for bootstrap-complete --dir= <1> +---- +<1> Specify the directory name that you generated the `install-config.yaml` file for +your cluster in. ++ +If the command exits without a `FATAL` warning, your production control plane +has initialized. \ No newline at end of file diff --git a/modules/installation-aws-upi-installation.adoc b/modules/installation-aws-upi-installation.adoc new file mode 100644 index 000000000000..682eb3428222 --- /dev/null +++ b/modules/installation-aws-upi-installation.adoc @@ -0,0 +1,90 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-aws-upi-installation-{context}"] += Completing an AWS UPI installation + +After you start the {product-title} installation on Amazon Web Service (AWS) +user-provisioned infrastructure, remove the bootstrap node, reconcile the default +Machine and MachineSet definitions, and delete unused nodes + +.Prerequisites + +* Deploy the bootstrap node for an {product-title} cluster on user-provisioned AWS infrastructure. +* Install the `oc` CLI and log in. + +.Procedure + +. Delete the bootstrap resources. If you used the CloudFormation template, +link:https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-console-delete-stack.html[delete its stack]: ++ +---- +$ aws cloudformation delete-stack --stack-name <1> +---- +<1> `` is the name of your bootstrap stack. + +//// +. View the list of machines in the `openshift-machine-api` namespace: ++ +---- +$ oc get machines --namespace openshift-machine-api +NAME INSTANCE STATE TYPE REGION ZONE AGE +test-tkh7l-master-0 m4.xlarge us-east-2 us-east-2a 9m22s +test-tkh7l-master-1 m4.xlarge us-east-2 us-east-2b 9m22s +test-tkh7l-master-2 m4.xlarge us-east-2 us-east-2c 9m21s +test-tkh7l-worker-us-east-2a-qjcxq m4.large us-east-2 us-east-2a 8m6s +test-tkh7l-worker-us-east-2b-nq8zs m4.large us-east-2 us-east-2b 8m6s +test-tkh7l-worker-us-east-2c-ww6c6 m4.large us-east-2 us-east-2c 8m7s +---- ++ +Note the `NAME` of each node. Because you manually deployed control plane +nodes, the master machines are not controlled by the Machine API. Similarly, +the worker machines are not backed by AWS instances on your subnet. You delete +each of these machines. + +. Delete each of the listed machines: ++ +---- +$ oc delete machine --namespace openshift-machine-api <1> +machine.machine.openshift.io "" deleted +---- +<1> Specify the name of a master or worker node to delete. +//// + +. Review the pending certificate signing requests (CSRs) and ensure that the +requests are for the machines that you added to the cluster: ++ +---- +$ oc get csr + +NAME AGE REQUESTOR CONDITION +csr-8b2br 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Approved,Issued +csr-8vnps 15m system:serviceaccount:openshift-machine-config-operator:node-bootstrapper Approved,Issued +csr-b96j4 25s system:node:ip-10-0-52-215.us-east-2.compute.internal Approved,Issued +csr-bfd72 5m26s system:node:ip-10-0-50-126.us-east-2.compute.internal Pending +csr-c57lv 5m26s system:node:ip-10-0-95-157.us-east-2.compute.internal Pending +---- + +. Approve the CSRs for your cluster machines. +** To approve them individually, run the following command for each valid +CSR: ++ +---- +$ oc adm certificate approve <1> +---- +<1> `` is the name of a CSR from the list of current CSRs. + +** If all the CSRs are valid, approve them all by running the following +command: ++ +---- +$ oc get csr -ojson | jq -r '.items[] | select(.status == {} ) | .metadata.name' | xargs oc adm certificate approve +---- + +. Complete the cluster installation: ++ +---- +$ ./openshift-install wait-for install-complete +INFO Waiting up to 30m0s for the cluster to initialize... +---- diff --git a/modules/installation-aws-upi-requirements.adoc b/modules/installation-aws-upi-requirements.adoc new file mode 100644 index 000000000000..3d67a6d95ae8 --- /dev/null +++ b/modules/installation-aws-upi-requirements.adoc @@ -0,0 +1,523 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-aws-upi-requirements-{context}"] += Required AWS UPI infrastructure components + +To install {product-title} on user-provisioned infrastructure in +Amazon Web Services (AWS), you must manually create both the machines and their +supporting infrastructure. + +You can use the provided CloudFormation templates to create this infrastructure, +you can manually create the components, or you can reuse existing infrastructure +that meets the cluster requirements. Review the CloudFormation templates +for more details about how the components interrelate. + +[id="installation-aws-upi-cluster-machines-{context}"] +== Cluster machines + +You need `AWS::EC2::Instance` objects for the following machines: + +* A bootstrap machine. This machine is required during installation, but you +can remove it after your cluster deploys. +* At least three control plane machines. The control plane machines are not +governed by a MachineSet. +* Compute machines. You can create these worker machines manually during the +cluster installation process. + +//// +You can also create and control them by using a MachineSet after your +control plane initializes and you can access the cluster API by using the `oc` +command line interface. +//// + +You can use the following instance types for the cluster machines: + +.Valid instance types for machines + +[cols="2a,2a,2a,2a",options="header"] +|=== + +|Instance type +|Bootstrap +|Control plane +|Compute + +|`i3.large` +|x +| +| + +| `m4.large` +| +| +|x + +| `m4.xlarge` +| +|x +|x + +| `m4.2xlarge` +| +|x +|x + +| `m4.4xlarge` +| +|x +|x + +| `m4.8xlarge` +| +|x +|x + +| `m4.10xlarge` +| +|x +|x + +| `m4.16xlarge` +| +|x +|x + +| `c4.large` +| +| +|x + +| `c4.xlarge` +| +| +|x + +| `c4.2xlarge` +| +|x +|x + +| `c4.4xlarge` +| +|x +|x + +| `c4.8xlarge` +| +|x +|x + +| `r4.large` +| +| +|x + +| `r4.xlarge` +| +|x +|x + +| `r4.2xlarge` +| +|x +|x + +| `r4.4xlarge` +| +|x +|x + +| `r4.8xlarge` +| +|x +|x + +| `r4.16xlarge` +| +|x +|x + +|=== + +[id="installation-aws-upi-other-infrastructure-{context}"] +== Other infrastructure components + +* A VPC +* DNS entries +* Load balancers and listeners +* A Route53 zone +* Security groups +* IAM roles +* S3 buckets + +.Required VPC components + +You need to provide a suitable VPC and subnets that allow communication to your +machines. + +[cols="2a,7a,3a,3a",options="header"] +|=== + +|Component +|AWS type +2+|Description + +|VPC +|* `AWS::EC2::VPC` +* `AWS::EC2::VPCEndpoint` +2+|You must provide a public VPC for the cluster to use. The VPC requires an +endpoint that references the route tables for each subnet. + +|Public subnets +|* `AWS::EC2::Subnet` +* `AWS::EC2::SubnetNetworkAclAssociation` +2+|Your VPC must have public subnets for between 1 and 3 availability zones +and associate them with appropriate ingress rules. + +|Internet gateway +| +* `AWS::EC2::InternetGateway` +* `AWS::EC2::VPCGatewayAttachment` +* `AWS::EC2::RouteTable` +* `AWS::EC2::Route` +* `PublicSubnetRouteTableAssociation` +* `AWS::EC2::NatGateway` +* `AWS::EC2::EIP` +2+|You must have a public internet gateway, with public routes, attached to the +VPC. Each public subnet must also be attached to the route and have a NAT +gateway and EIP address. + +.7+|Network access control +.7+| * `AWS::EC2::NetworkAcl` +* `AWS::EC2::NetworkAclEntry` +2+|You must allow the VPC to access the following ports: +h|Port +h|Reason + +|`80` +|Inbound HTTP traffic + +|`443` +|Inbound HTTPS traffic + +|`22` +|Inbound SSH traffic + +|`1024` - `65535` +|Inbound ephemeral traffic + +|`0` - `65535` +|Outbound ephemeral traffic + + +|Private subnets +|* `AWS::EC2::Subnet` +* `AWS::EC2::RouteTable` +* `AWS::EC2::SubnetRouteTableAssociation` +2+|Your VPC can have a private subnets. The provided CloudFormation templates +can create private subnets for between 1 and 3 availability zones. +If you use private subnets, you must provide appropriate routes and tables +for them. + +|=== + + +.Required DNS and load balancing components + +Your DNS and load balancer configuration needs to use a public hosted zone and +can use a private hosted zone similar to the IPI installation method. You must +create a DNS entry that resolves to your load balancer. An entry for +`api..` must point to the external load balancer, and an +entry for `api-int..` must point to the internal load +balancer. + +The cluster also requires load balancers and listeners for port 6443, which are +required for the Kubernetes API and its extensions, and port 22623, which are +required for the Ignition config files for new machines. The targets will be the +master nodes. Port 6443 must be accessible to both clients external to the +cluster and nodes within the cluster. Port 22623 must be accessible to nodes +within the cluster. + + +[cols="2a,2a,8a",options="header"] +|=== + +|Component +|AWS type +|Description + +|DNS +|`AWS::Route53::HostedZone` +|The hosted zone for your internal DNS. + +|etcd record sets +|`AWS::Route53::RecordSet` +|The registration records for etcd for your control plane machines. + +|Public load balancer +|`AWS::ElasticLoadBalancingV2::LoadBalancer` +|The load balancer for your public subnets. + +|External API server record +|`AWS::Route53::RecordSetGroup` +|Alias records for the external API server. + +|External listener +|`AWS::ElasticLoadBalancingV2::Listener` +|A listener on port 6443 for the external load balancer. + +|External target group +|`AWS::ElasticLoadBalancingV2::TargetGroup` +|The target group for the external load balancer. + +|Private load balancer +|`AWS::ElasticLoadBalancingV2::LoadBalancer` +|The load balancer for your private subnets. + +|Internal API server record +|`AWS::Route53::RecordSetGroup` +|Alias records for the internal API server. + +|Internal listener +|`AWS::ElasticLoadBalancingV2::Listener` +|A listener on port 26443 for the internal load balancer. + +|Internal target group +|`AWS::ElasticLoadBalancingV2::TargetGroup` +|The target group for the Internal load balancer. + +|Internal listener +|`AWS::ElasticLoadBalancingV2::Listener` +|A listener on port 26443 for the internal load balancer. + +|Internal target group +|`AWS::ElasticLoadBalancingV2::TargetGroup` +|The target group for the Internal load balancer. + +|=== + +.Security groups + +The control plane and worker machines require access to the following ports: + +[cols="2a,2a,2a,2a",options="header"] +|=== + +|Group +|Type +|IP Protocol +|Port range + + +.4+|MasterSecurityGroup +.4+|`AWS::EC2::SecurityGroup` +|`icmp` +|`0` + +|`tcp` +|`22` + +|`tcp` +|`6443` + +|`tcp` +|`22623` + +.2+|WorkerSecurityGroup +.2+|`AWS::EC2::SecurityGroup` +|`icmp` +|`0` + +|`tcp` +|`22` + + +.2+|BootstrapSecurityGroup +.2+|`AWS::EC2::SecurityGroup` + +|`tcp` +|`22` + +|`tcp` +|`19531` + +|=== + +.Control plane ingress + +The control plane machines require the following ingress groups. Each ingress group is +a `AWS::EC2::SecurityGroupIngress` resource. + +[cols="2a,5a,2a,2a",options="header"] +|=== + +|Ingress group +|Description +|IP protocol +|Port range + + +|`MasterIngressEtcd` +|etcd +|`tcp` +|`2379`- `2380` + +|`MasterIngressVxlan` +|Vxlan packets +|`udp` +|`4789` + +|`MasterIngressWorkerVxlan` +|Vxlan packets +|`udp` +|`4789` + +|`MasterIngressInternal` +|Internal cluster communication +|`tcp` +|`9000` - `9999` + +|`MasterIngressWorkerInternal` +|Internal cluster communication +|`tcp` +|`9000` - `9999` + +|`MasterIngressKube` +|Kubernetes kubelet, scheduler and controller manager +|`tcp` +|`10250` - `10259` + +|`MasterIngressWorkerKube` +|Kubernetes kubelet, scheduler and controller manager +|`tcp` +|`10250` - `10259` + +|`MasterIngressIngressServices` +|Kubernetes ingress services +|`tcp` +|`30000` - `32767` + +|`MasterIngressWorkerIngressServices` +|Kubernetes ingress services +|`tcp` +|`30000` - `32767` + +|=== + + +.Worker ingress + +The worker machines require the following ingress groups. Each ingress group is +a `AWS::EC2::SecurityGroupIngress` resource. + +[cols="2a,5a,2a,2a",options="header"] +|=== + +|Ingress group +|Description +|IP protocol +|Port range + + +|`WorkerIngressVxlan` +|Vxlan packets +|`udp` +|`4789` + +|`WorkerIngressWorkerVxlan` +|Vxlan packets +|`udp` +|`4789` + +|`WorkerIngressInternal` +|Internal cluster communication +|`tcp` +|`9000` - `9999` + +|`WorkerIngressWorkerInternal` +|Internal cluster communication +|`tcp` +|`9000` - `9999` + +|`WorkerIngressKube` +|Kubernetes kubelet, scheduler and controller manager +|`tcp` +|`10250` + +|`WorkerIngressWorkerKube` +|Kubernetes kubelet, scheduler and controller manager +|`tcp` +|`10250` + +|`WorkerIngressIngressServices` +|Kubernetes ingress services +|`tcp` +|`30000` - `32767` + +|`WorkerIngressWorkerIngressServices` +|Kubernetes ingress services +|`tcp` +|`30000` - `32767` + +|=== + + +.Roles and instance profiles + +You must grant the machines permissions in AWS. The provided CloudFormation +templates grant the machines permission the following `AWS::IAM::Role` objects +and provide a `AWS::IAM::InstanceProfile` for each set of roles. If you do +not use the templates, you can grant the machines the following broad permissions +or the following individual permissions. + +[cols="2a,2a,2a,2a",options="header"] +|=== + +|Role +|Effect +|Action +|Resource + +.4+|Master +|`Allow` +|`ec2:*` +|`*` + +|`Allow` +|`elasticloadbalancing:*` +|`*` + +|`Allow` +|`iam:PassRole` +|`*` + +|`Allow` +|`s3:GetObject` +|`*` + +|Worker +|`Allow` +|`ec2:Describe*` +|`*` + + +.3+|Bootstrap +|`Allow` +|`ec2:Describe*` +|`*` + +|`Allow` +|`ec2:AttachVolume` +|`*` + +|`Allow` +|`ec2:DetachVolume` +|`*` + +|`Allow` +|`s3:GetObject` +|`*` + +|=== diff --git a/modules/installation-aws-upi-rhcos-ami.adoc b/modules/installation-aws-upi-rhcos-ami.adoc new file mode 100644 index 000000000000..798d14e81fb2 --- /dev/null +++ b/modules/installation-aws-upi-rhcos-ami.adoc @@ -0,0 +1,66 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-aws-upi-rhcos-ami-{context}"] += {op-system} AMIs for the AWS UPI infrastructure + +You must use a valid {op-system-first} AMI for your Amazon Web Services +(AWS) zone for your {product-title} nodes. + +.{op-system} AMIs + +[cols="2a,2a",options="header"] +|=== + +|AWS zone +|AWS AMI + +|`ap-northeast-1` +|`ami-06153e8d5cf577a81` + +|`ap-northeast-2` +|`ami-0b75c8dbcc07363b9` + +|`ap-south-1` +|`ami-08a07f507cd20aa23` + +|`ap-southeast-1` +|`ami-030cd605c9d3eec03` + +|`ap-southeast-2` +|`ami-0140da25e0be718b2` + +|`ca-central-1` +|`ami-027d77f6170fde18b` + +|`eu-central-1` +|`ami-089595a0f4d60a839` + +|`eu-west-1` +|`ami-063ce7cafbc2a16df` + +|`eu-west-2` +|`ami-06619611f27b0103a` + +|`eu-west-3` +|`ami-0ec28a762270a0d8f` + +|`sa-east-1` +|`ami-0845523ad10519ac4` + +|`us-east-1` +|`ami-0678c4c4a53cd4680` + +|`us-east-2` +|`ami-07e0e0e0035b5a3fe` + +|`us-west-1` +|`ami-0c58a195b65f5250f` + +|`us-west-2` +|`ami-01e2758deb3135fb0` + +|=== + + diff --git a/modules/installation-cloudformation-bootstrap.adoc b/modules/installation-cloudformation-bootstrap.adoc new file mode 100644 index 000000000000..762d00e71682 --- /dev/null +++ b/modules/installation-cloudformation-bootstrap.adoc @@ -0,0 +1,224 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-bootstrap-{context}"] += CloudFormation template for the bootstrap machine + +You can use the following CloudFormation template to deploy the bootstrap +machine that you need for your {product-title} cluster. + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Openshift Cluster UPI Bootstrap (EC2 Instance, Security Groups and IAM) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, unique cluster ID used to tag cloud resources and identify items owned/used by the cluster. + Type: String + RhcosAmi: + Description: Current RHEL CoreOS AMI to use for boostrap + Type: AWS::EC2::Image::Id + AllowedBootstrapSshCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|1[0-9]|2[0-9]|3[0-2]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/0-32 + Default: 0.0.0.0/0 + Description: CIDR block to allow SSH access to the bootstrap node + Type: String + PublicSubnet: + Description: The public subnet to launch the bootstrap node into + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID (for registering temporary rules) + Type: AWS::EC2::SecurityGroup::Id + VpcId: + Description: The VPC created resources will belong. + Type: AWS::EC2::VPC::Id + BootstrapIgnitionLocation: + Default: s3://my-s3-bucket/bootstrap.ign + Description: Location to fetch bootstrap ignition from. (Recommend to use the autocreated cf-templates bucket.) + Type: String + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration (requires Lambda ARN parameter to be supplied)? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda + Type: String + ExternalApiTargetGroupArn: + Description: ARN for external API load balancer target group + Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Host Information" + Parameters: + - RhcosAmi + - BootstrapIgnitionLocation + - MasterSecurityGroupId + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - PublicSubnet + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn + - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + AllowedBootstrapSshCidr: + default: "Allowed SSH Source" + PublicSubnet: + default: "Public Subnet" + RhcosAmi: + default: "RHEL CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Bootstrap Ignition Source" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterELB: + default: "Use Provided ELB Automation" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + +Resources: + BootstrapIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "bootstrap", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:Describe*" + Resource: "*" + - Effect: "Allow" + Action: "ec2:AttachVolume" + Resource: "*" + - Effect: "Allow" + Action: "ec2:DetachVolume" + Resource: "*" + - Effect: "Allow" + Action: "s3:GetObject" + Resource: "*" + + BootstrapInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Path: "/" + Roles: + - Ref: "BootstrapIamRole" + + BootstrapSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Cluster Bootstrap Security Group + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref AllowedBootstrapSshCidr + - IpProtocol: tcp + ToPort: 19531 + FromPort: 19531 + CidrIp: 0.0.0.0/0 + VpcId: !Ref VpcId + + BootstrapInstance: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref BootstrapInstanceProfile + InstanceType: "i3.large" + NetworkInterfaces: + - AssociatePublicIpAddress: "true" + DeviceIndex: "0" + GroupSet: + - !Ref "BootstrapSecurityGroup" + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "PublicSubnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"replace":{"source":"${S3Loc}","verification":{}}},"timeouts":{},"version":"2.1.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + S3Loc: !Ref BootstrapIgnitionLocation + } + + RegisterBootstrapApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + + RegisterBootstrapInternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt BootstrapInstance.PrivateIp + +Outputs: + BootstrapInstanceId: + Description: Bootstrap Instance ID + Value: !Ref BootstrapInstance + + BootstrapPublicIp: + Description: The bootstrap node public IP address + Value: !GetAtt BootstrapInstance.PublicIp + + BootstrapPrivateIp: + Description: The bootstrap node private IP address + Value: !GetAtt BootstrapInstance.PrivateIp +---- \ No newline at end of file diff --git a/modules/installation-cloudformation-control-plane.adoc b/modules/installation-cloudformation-control-plane.adoc new file mode 100644 index 000000000000..ca22b4d8147d --- /dev/null +++ b/modules/installation-cloudformation-control-plane.adoc @@ -0,0 +1,374 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-control-plane-{context}"] += CloudFormation template for control plane machines + +You can use the following CloudFormation template to deploy the control plane +machines that you need for your {product-title} cluster. + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Openshift Cluster UPI Node Launch (EC2 master instances) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, unique cluster ID used to tag nodes for the kubelet cloud provider. + Type: String + RhcosAmi: + Description: Current RHEL CoreOS AMI to use for boostrap + Type: AWS::EC2::Image::Id + AutoRegisterDNS: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke DNS etcd registration (requires Hosted Zone info provided)? + Type: String + PrivateHostedZoneId: + Description: The Route53 private zone ID to register the etcd targets with (e.g Z21IXYZABCZ2A4) + Type: String + PrivateHostedZoneName: + Description: The Route53 zone to register the targets with (No trailing dot - e.g. cluster.mycorp.com) + Type: String + Master0Subnet: + Description: The subnets (recommend private) to launch the master nodes into + Type: AWS::EC2::Subnet::Id + Master1Subnet: + Description: The subnets (recommend private) to launch the master nodes into + Type: AWS::EC2::Subnet::Id + Master2Subnet: + Description: The subnets (recommend private) to launch the master nodes into + Type: AWS::EC2::Subnet::Id + MasterSecurityGroupId: + Description: The master security group ID to associate with master nodes. + Type: AWS::EC2::SecurityGroup::Id + IgnitionLocation: + Default: https://api-int.$CLUSTER_NAME.$DOMAIN:22623/config/master + Description: Location to fetch bootstrap ignition from. (Recommend to use the autocreated ignition config location.) + Type: String + CertificateAuthorities: + Default: data:text/plain;charset=utf-8;base64,ABC...xYz== + Description: Base64 encoded certificate authority string to use. + Type: String + MasterInstanceProfileName: + Description: IAM profile to associate with master nodes. + Type: String + MasterInstanceType: + Default: m4.xlarge + Type: String + AllowedValues: + - "m4.xlarge" + - "m4.2xlarge" + - "m4.4xlarge" + - "m4.8xlarge" + - "m4.10xlarge" + - "m4.16xlarge" + - "c4.2xlarge" + - "c4.4xlarge" + - "c4.8xlarge" + - "r4.xlarge" + - "r4.2xlarge" + - "r4.4xlarge" + - "r4.8xlarge" + - "r4.16xlarge" + AutoRegisterELB: + Default: "yes" + AllowedValues: + - "yes" + - "no" + Description: Do you want to invoke NLB registration (requires Lambda ARN parameter to be supplied)? + Type: String + RegisterNlbIpTargetsLambdaArn: + Description: ARN for NLB IP target registration lambda (from cluster_infra_upi.yaml; otherwise select "no" for AutoRegisterELB) + Type: String + ExternalApiTargetGroupArn: + Description: ARN for external API load balancer target group (from cluster_infra_upi.yaml; otherwise select "no" for AutoRegisterELB) + Type: String + InternalApiTargetGroupArn: + Description: ARN for internal API load balancer target group (from cluster_infra_upi.yaml; otherwise select "no" for AutoRegisterELB) + Type: String + InternalServiceTargetGroupArn: + Description: ARN for internal service load balancer target group (from cluster_infra_upi.yaml; otherwise select "no" for AutoRegisterELB) + Type: String + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Host Information" + Parameters: + - MasterInstanceType + - RhcosAmi + - IgnitionLocation + - CertificateAuthorities + - MasterSecurityGroupId + - MasterInstanceProfileName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - AllowedBootstrapSshCidr + - Master0Subnet + - Master1Subnet + - Master2Subnet + - Label: + default: "DNS" + Parameters: + - AutoRegisterDNS + - PrivateHostedZoneName + - PrivateHostedZoneId + - Label: + default: "Load Balancer Automation" + Parameters: + - AutoRegisterELB + - RegisterNlbIpTargetsLambdaArn + - ExternalApiTargetGroupArn + - InternalApiTargetGroupArn + - InternalServiceTargetGroupArn + ParameterLabels: + VpcId: + default: "VPC ID" + Master0Subnet: + default: "Master-0 Subnet" + Master1Subnet: + default: "Master-1 Subnet" + Master2Subnet: + default: "Master-2 Subnet" + MasterInstanceType: + default: "Master Instance Type" + MasterInstanceProfileName: + default: "Master Instance Profile Name" + RhcosAmi: + default: "RHEL CoreOS AMI ID" + BootstrapIgnitionLocation: + default: "Master Ignition Source" + CertificateAuthorities: + default: "Ignition CA String" + MasterSecurityGroupId: + default: "Master Security Group ID" + AutoRegisterDNS: + default: "Use Provided DNS Automation" + AutoRegisterELB: + default: "Use Provided ELB Automation" + PrivateHostedZoneName: + default: "Private Hosted Zone Name" + PrivateHostedZoneId: + default: "Private Hosted Zone ID" + +Conditions: + DoRegistration: !Equals ["yes", !Ref AutoRegisterELB] + DoDns: !Equals ["yes", !Ref AutoRegisterDNS] + +Resources: + Master0: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master0Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster0: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + RegisterMaster0InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master0.PrivateIp + + Master1: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master1Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster1: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + RegisterMaster1InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master1.PrivateIp + + Master2: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref MasterInstanceProfileName + InstanceType: !Ref MasterInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "MasterSecurityGroupId" + SubnetId: !Ref "Master2Subnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + + RegisterMaster2: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref ExternalApiTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalApiTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalApiTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + RegisterMaster2InternalServiceTarget: + Condition: DoRegistration + Type: Custom::NLBRegister + Properties: + ServiceToken: !Ref RegisterNlbIpTargetsLambdaArn + TargetArn: !Ref InternalServiceTargetGroupArn + TargetIp: !GetAtt Master2.PrivateIp + + EtcdSrvRecords: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["_etcd-server-ssl._tcp", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]]], + ] + - !Join [ + " ", + ["0 10 2380", !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]]], + ] + TTL: 60 + Type: SRV + + Etcd0Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-0", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master0.PrivateIp + TTL: 60 + Type: A + + Etcd1Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-1", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master1.PrivateIp + TTL: 60 + Type: A + + Etcd2Record: + Condition: DoDns + Type: AWS::Route53::RecordSet + Properties: + HostedZoneId: !Ref PrivateHostedZoneId + Name: !Join [".", ["etcd-2", !Ref PrivateHostedZoneName]] + ResourceRecords: + - !GetAtt Master2.PrivateIp + TTL: 60 + Type: A + +Outputs: + PrivateIPs: + Description: The control-plane node private IP addresses + Value: + !Join [ + ",", + [!GetAtt Master0.PrivateIp, !GetAtt Master1.PrivateIp, !GetAtt Master2.PrivateIp] + ] +---- \ No newline at end of file diff --git a/modules/installation-cloudformation-dns.adoc b/modules/installation-cloudformation-dns.adoc new file mode 100644 index 000000000000..915646c29089 --- /dev/null +++ b/modules/installation-cloudformation-dns.adoc @@ -0,0 +1,394 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-dns-{context}"] += CloudFormation template for the network and load balancers + +You can use the following CloudFormation template to deploy the networking +objects and load balancers that you need for your {product-title} cluster. + + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Openshift Cluster UPI Network Elements (Route53 & LBs) + +Parameters: + ClusterName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Cluster name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, representative cluster name to use for hostnames, etc. + Type: String + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, unique cluster ID used to tag cloud resources and identify items owned/used by the cluster. + Type: String + HostedZoneId: + Description: The Route53 public zone ID to register the targets with (e.g Z21IXYZABCZ2A4) + Type: String + HostedZoneName: + Description: The Route53 zone to register the targets with (No trailing dot - e.g. mycorp.com) + Type: String + Default: "example.com" + PublicSubnets: + Description: The internet-facing subnets + Type: List + PrivateSubnets: + Description: The internal subnets + Type: List + VpcId: + Description: The VPC created resources will belong. + Type: AWS::EC2::VPC::Id + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - ClusterName + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - PublicSubnets + - PrivateSubnets + - Label: + default: "DNS" + Parameters: + - HostedZoneName + - HostedZoneId + ParameterLabels: + ClusterName: + default: "Cluster Name" + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + PublicSubnets: + default: "Public Subnets" + PrivateSubnets: + default: "Private Subnets" + HostedZoneName: + default: "Public Hosted Zone Name" + HostedZoneId: + default: "Public Hosted Zone ID" + +Resources: + ExtApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", ["api-ext", !Ref ClusterName, !Ref "AWS::Region"]] + IpAddressType: ipv4 + Subnets: !Ref PublicSubnets + Type: network + + IntApiElb: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Join ["-", ["api-int", !Ref ClusterName, !Ref "AWS::Region"]] + Scheme: internal + IpAddressType: ipv4 + Subnets: !Ref PrivateSubnets + Type: network + + IntDns: + Type: "AWS::Route53::HostedZone" + Properties: + HostedZoneConfig: + Comment: "Managed by CloudFormation" + Name: !Join [".", [!Ref ClusterName, !Ref HostedZoneName]] + HostedZoneTags: + - Key: Name + Value: !Join ["-", [!Ref InfrastructureName, "int"]] + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "owned" + VPCs: + - VPCId: !Ref VpcId + VPCRegion: !Ref "AWS::Region" + + ExternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref HostedZoneId + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt ExtApiElb.CanonicalHostedZoneID + DNSName: !GetAtt ExtApiElb.DNSName + + InternalApiServerRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + Comment: Alias record for the API server + HostedZoneId: !Ref IntDns + RecordSets: + - Name: + !Join [ + ".", + ["api", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + - Name: + !Join [ + ".", + ["api-int", !Ref ClusterName, !Join ["", [!Ref HostedZoneName, "."]]], + ] + Type: A + AliasTarget: + HostedZoneId: !GetAtt IntApiElb.CanonicalHostedZoneID + DNSName: !GetAtt IntApiElb.DNSName + + ExternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: ExternalApiTargetGroup + LoadBalancerArn: + Ref: ExtApiElb + Port: 6443 + Protocol: TCP + + ExternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalApiListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalApiTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 6443 + Protocol: TCP + + InternalApiTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 6443 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + InternalServiceInternalListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: + Ref: InternalServiceTargetGroup + LoadBalancerArn: + Ref: IntApiElb + Port: 22623 + Protocol: TCP + + InternalServiceTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Port: 22623 + Protocol: TCP + TargetType: ip + VpcId: + Ref: VpcId + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 60 + + RegisterTargetLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "nlb", "lambda", "role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalApiTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref InternalServiceTargetGroup + - Effect: "Allow" + Action: + [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource: !Ref ExternalApiTargetGroup + + RegisterNlbIpTargets: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterTargetLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + elb = boto3.client('elbv2') + if event['RequestType'] == 'Delete': + elb.deregister_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + elif event['RequestType'] == 'Create': + elb.register_targets(TargetGroupArn=event['ResourceProperties']['TargetArn'],Targets=[{'Id': event['ResourceProperties']['TargetIp']}]) + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['TargetArn']+event['ResourceProperties']['TargetIp']) + Runtime: "python3.7" + Timeout: 120 + + RegisterSubnetTagsLambdaIamRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Join ["-", [!Ref InfrastructureName, "subnet-tags-lambda-role"]] + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + Path: "/" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "subnet-tagging-policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + [ + "ec2:DeleteTags", + "ec2:CreateTags" + ] + Resource: "arn:aws:ec2:*:*:subnet/*" + - Effect: "Allow" + Action: + [ + "ec2:DescribeSubnets", + "ec2:DescribeTags" + ] + Resource: "*" + + RegisterSubnetTags: + Type: "AWS::Lambda::Function" + Properties: + Handler: "index.handler" + Role: + Fn::GetAtt: + - "RegisterSubnetTagsLambdaIamRole" + - "Arn" + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + def handler(event, context): + ec2_client = boto3.client('ec2') + if event['RequestType'] == 'Delete': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.delete_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName']}]); + elif event['RequestType'] == 'Create': + for subnet_id in event['ResourceProperties']['Subnets']: + ec2_client.create_tags(Resources=[subnet_id], Tags=[{'Key': 'kubernetes.io/cluster/' + event['ResourceProperties']['InfrastructureName'], 'Value': 'shared'}]); + responseData = {} + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, event['ResourceProperties']['InfrastructureName']+event['ResourceProperties']['Subnets'][0]) + Runtime: "python3.7" + Timeout: 120 + + RegisterPublicSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PublicSubnets + + RegisterPrivateSubnetTags: + Type: Custom::SubnetRegister + Properties: + ServiceToken: !GetAtt RegisterSubnetTags.Arn + InfrastructureName: !Ref InfrastructureName + Subnets: !Ref PrivateSubnets + +Outputs: + PrivateHostedZoneId: + Description: Hosted zone ID for the private DNS - needed for private records + Value: !Ref IntDns + ExternalApiLoadBalancerName: + Description: Full name of the External API load balancer created. + Value: !GetAtt ExtApiElb.LoadBalancerFullName + InternalApiLoadBalancerName: + Description: Full name of the Internal API load balancer created. + Value: !GetAtt IntApiElb.LoadBalancerFullName + ApiServerDnsName: + Description: Full hostname of the API server - Needed for ignition configs + Value: !Join [".", ["api-int", !Ref ClusterName, !Ref HostedZoneName]] + RegisterNlbIpTargetsLambda: + Description: Lambda ARN useful to help register/deregister IP targets for these load balancers + Value: !GetAtt RegisterNlbIpTargets.Arn + ExternalApiTargetGroupArn: + Description: ARN of External API target group + Value: !Ref ExternalApiTargetGroup + InternalApiTargetGroupArn: + Description: ARN of Internal API target group + Value: !Ref InternalApiTargetGroup + InternalServiceTargetGroupArn: + Description: ARN of internal service target group + Value: !Ref InternalServiceTargetGroup +---- \ No newline at end of file diff --git a/modules/installation-cloudformation-security.adoc b/modules/installation-cloudformation-security.adoc new file mode 100644 index 000000000000..3f89d4cdacf1 --- /dev/null +++ b/modules/installation-cloudformation-security.adoc @@ -0,0 +1,348 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-security-{context}"] += CloudFormation template for security objects + +You can use the following CloudFormation template to deploy the security objects +that you need for your {product-title} cluster. + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Openshift Cluster UPI Security Elements (Security Groups & IAM) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, unique cluster ID used to tag cloud resources and identify items owned/used by the cluster. + Type: String + VpcCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24 + Default: 10.0.0.0/16 + Description: CIDR block for VPC + Type: String + VpcId: + Description: The VPC created resources will belong. + Type: AWS::EC2::VPC::Id + PrivateSubnets: + Description: The internal subnets + Type: List + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Cluster Information" + Parameters: + - InfrastructureName + - Label: + default: "Network Configuration" + Parameters: + - VpcId + - VpcCidr + - PrivateSubnets + ParameterLabels: + InfrastructureName: + default: "Infrastructure Name" + VpcId: + default: "VPC ID" + VpcCidr: + default: "VPC CIDR" + PrivateSubnets: + default: "Private Subnets" + +Resources: + MasterSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Cluster Master Security Group + SecurityGroupIngress: + - IpProtocol: icmp + FromPort: 0 + ToPort: 0 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + ToPort: 6443 + FromPort: 6443 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22623 + ToPort: 22623 + CidrIp: !Ref VpcCidr + VpcId: !Ref VpcId + + WorkerSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Cluster Worker Security Group + SecurityGroupIngress: + - IpProtocol: icmp + FromPort: 0 + ToPort: 0 + CidrIp: !Ref VpcCidr + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCidr + VpcId: !Ref VpcId + + MasterIngressEtcd: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: etcd + FromPort: 2379 + ToPort: 2380 + IpProtocol: tcp + + MasterIngressVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + MasterIngressWorkerVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + MasterIngressInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + MasterIngressWorkerInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + MasterIngressKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes kubelet, scheduler and controller manager + FromPort: 10250 + ToPort: 10259 + IpProtocol: tcp + + MasterIngressWorkerKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes kubelet, scheduler and controller manager + FromPort: 10250 + ToPort: 10259 + IpProtocol: tcp + + MasterIngressIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + MasterIngressWorkerIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt MasterSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + WorkerIngressVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + WorkerIngressWorkerVxlan: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Vxlan packets + FromPort: 4789 + ToPort: 4789 + IpProtocol: udp + + WorkerIngressInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + WorkerIngressWorkerInternal: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal cluster communication + FromPort: 9000 + ToPort: 9999 + IpProtocol: tcp + + WorkerIngressKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes secure kubelet port + FromPort: 10250 + ToPort: 10250 + IpProtocol: tcp + + WorkerIngressWorkerKube: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Internal Kubernetes communication + FromPort: 10250 + ToPort: 10250 + IpProtocol: tcp + + WorkerIngressIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt WorkerSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + WorkerIngressWorkerIngressServices: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !GetAtt WorkerSecurityGroup.GroupId + SourceSecurityGroupId: !GetAtt MasterSecurityGroup.GroupId + Description: Kubernetes ingress services + FromPort: 30000 + ToPort: 32767 + IpProtocol: tcp + + MasterIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "master", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:*" + Resource: "*" + - Effect: "Allow" + Action: "elasticloadbalancing:*" + Resource: "*" + - Effect: "Allow" + Action: "iam:PassRole" + Resource: "*" + - Effect: "Allow" + Action: "s3:GetObject" + Resource: "*" + + MasterInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - Ref: "MasterIamRole" + + WorkerIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "ec2.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: !Join ["-", [!Ref InfrastructureName, "worker", "policy"]] + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "ec2:Describe*" + Resource: "*" + + WorkerInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + Roles: + - Ref: "WorkerIamRole" + +Outputs: + MasterSecurityGroupId: + Description: Master Security Group ID + Value: !GetAtt MasterSecurityGroup.GroupId + + WorkerSecurityGroupId: + Description: Worker Security Group ID + Value: !GetAtt WorkerSecurityGroup.GroupId + + MasterInstanceProfile: + Description: Master IAM Instance Profile + Value: !Ref MasterInstanceProfile + + WorkerInstanceProfile: + Description: Worker IAM Instance Profile + Value: !Ref WorkerInstanceProfile +---- \ No newline at end of file diff --git a/modules/installation-cloudformation-vpc.adoc b/modules/installation-cloudformation-vpc.adoc new file mode 100644 index 000000000000..f41d84b7c7cd --- /dev/null +++ b/modules/installation-cloudformation-vpc.adoc @@ -0,0 +1,382 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-vpc-{context}"] += CloudFormation template for the VPC + +You can use the following CloudFormation template to deploy the VPC that +you need for your {product-title} cluster. + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Best Practice VPC with 1-3 AZs + +Parameters: + VpcCidr: + AllowedPattern: ^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(1[6-9]|2[0-4]))$ + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/16-24 + Default: 10.0.0.0/16 + Description: CIDR block for VPC + Type: String + AvailabilityZoneCount: + ConstraintDescription: "The number of availability zones (Min: 1, Max: 3)" + MinValue: 1 + MaxValue: 3 + Default: 1 + Description: "How many AZs to create VPC subnets for (Min: 1, Max: 3)" + Type: Number + SubnetBits: + ConstraintDescription: CIDR block parameter must be in the form x.x.x.x/19-27 + MinValue: 5 + MaxValue: 13 + Default: 12 + Description: "Size of each subnet to create within the availability zones. (Min: 5 = /27, Max: 13 = /19)" + Type: Number + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Network Configuration" + Parameters: + - VpcCidr + - SubnetBits + - Label: + default: "Availability Zones" + Parameters: + - AvailabilityZoneCount + ParameterLabels: + AvailabilityZoneCount: + default: "Availability Zone Count" + VpcCidr: + default: "VPC CIDR" + SubnetBits: + default: "Bits Per Subnet" + +Conditions: + DoAz3: !Equals [3, !Ref AvailabilityZoneCount] + DoAz2: !Or [!Equals [2, !Ref AvailabilityZoneCount], Condition: DoAz3] + +Resources: + VPC: + Type: "AWS::EC2::VPC" + Properties: + EnableDnsSupport: "true" + EnableDnsHostnames: "true" + CidrBlock: !Ref VpcCidr + PublicSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [0, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [1, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PublicSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [2, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + InternetGateway: + Type: "AWS::EC2::InternetGateway" + GatewayToInternet: + Type: "AWS::EC2::VPCGatewayAttachment" + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + PublicRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PublicRoute: + Type: "AWS::EC2::Route" + DependsOn: GatewayToInternet + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + PublicSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PublicSubnet2 + RouteTableId: !Ref PublicRouteTable + PublicSubnetRouteTableAssociation3: + Condition: DoAz3 + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PublicSubnet3 + RouteTableId: !Ref PublicRouteTable + PublicNetworkAcl: + Type: "AWS::EC2::NetworkAcl" + Properties: + VpcId: !Ref VPC + InboundHTTPPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "100" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "80" + To: "80" + InboundHTTPSPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "101" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "443" + To: "443" + InboundSSHPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "102" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "22" + To: "22" + InboundEphemeralPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "103" + Protocol: "6" + RuleAction: allow + Egress: "false" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "1024" + To: "65535" + OutboundPublicNetworkAclEntry: + Type: "AWS::EC2::NetworkAclEntry" + Properties: + NetworkAclId: !Ref PublicNetworkAcl + RuleNumber: "100" + Protocol: "6" + RuleAction: allow + Egress: "true" + CidrBlock: 0.0.0.0/0 + PortRange: + From: "0" + To: "65535" + PublicSubnetNetworkAclAssociation: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Properties: + SubnetId: !Ref PublicSubnet + NetworkAclId: !Ref PublicNetworkAcl + PublicSubnetNetworkAclAssociation2: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PublicSubnet2 + NetworkAclId: !Ref PublicNetworkAcl + PublicSubnetNetworkAclAssociation3: + Type: "AWS::EC2::SubnetNetworkAclAssociation" + Condition: DoAz3 + Properties: + SubnetId: !Ref PublicSubnet3 + NetworkAclId: !Ref PublicNetworkAcl + PrivateSubnet: + Type: "AWS::EC2::Subnet" + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [3, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 0 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable: + Type: "AWS::EC2::RouteTable" + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Properties: + SubnetId: !Ref PrivateSubnet + RouteTableId: !Ref PrivateRouteTable + NAT: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Properties: + AllocationId: + "Fn::GetAtt": + - EIP + - AllocationId + SubnetId: !Ref PublicSubnet + EIP: + Type: "AWS::EC2::EIP" + Properties: + Domain: vpc + Route: + Type: "AWS::EC2::Route" + Properties: + RouteTableId: + Ref: PrivateRouteTable + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT + PrivateSubnet2: + Type: "AWS::EC2::Subnet" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [4, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 1 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable2: + Type: "AWS::EC2::RouteTable" + Condition: DoAz2 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation2: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz2 + Properties: + SubnetId: !Ref PrivateSubnet2 + RouteTableId: !Ref PrivateRouteTable2 + NAT2: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Condition: DoAz2 + Properties: + AllocationId: + "Fn::GetAtt": + - EIP2 + - AllocationId + SubnetId: !Ref PublicSubnet2 + EIP2: + Type: "AWS::EC2::EIP" + Condition: DoAz2 + Properties: + Domain: vpc + Route2: + Type: "AWS::EC2::Route" + Condition: DoAz2 + Properties: + RouteTableId: + Ref: PrivateRouteTable2 + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT2 + PrivateSubnet3: + Type: "AWS::EC2::Subnet" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + CidrBlock: !Select [5, !Cidr [!Ref VpcCidr, 6, !Ref SubnetBits]] + AvailabilityZone: !Select + - 2 + - Fn::GetAZs: !Ref "AWS::Region" + PrivateRouteTable3: + Type: "AWS::EC2::RouteTable" + Condition: DoAz3 + Properties: + VpcId: !Ref VPC + PrivateSubnetRouteTableAssociation3: + Type: "AWS::EC2::SubnetRouteTableAssociation" + Condition: DoAz3 + Properties: + SubnetId: !Ref PrivateSubnet3 + RouteTableId: !Ref PrivateRouteTable3 + NAT3: + DependsOn: + - GatewayToInternet + Type: "AWS::EC2::NatGateway" + Condition: DoAz3 + Properties: + AllocationId: + "Fn::GetAtt": + - EIP3 + - AllocationId + SubnetId: !Ref PublicSubnet3 + EIP3: + Type: "AWS::EC2::EIP" + Condition: DoAz3 + Properties: + Domain: vpc + Route3: + Type: "AWS::EC2::Route" + Condition: DoAz3 + Properties: + RouteTableId: + Ref: PrivateRouteTable3 + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NAT3 + S3Endpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: '*' + Action: + - '*' + Resource: + - '*' + RouteTableIds: + - !Ref PublicRouteTable + - !Ref PrivateRouteTable + - !If [DoAz2, !Ref PrivateRouteTable2, !Ref "AWS::NoValue"] + - !If [DoAz3, !Ref PrivateRouteTable3, !Ref "AWS::NoValue"] + ServiceName: !Join + - '' + - - com.amazonaws. + - !Ref 'AWS::Region' + - .s3 + VpcId: !Ref VPC + +Outputs: + VpcId: + Description: ID of the newly created VPC + Value: !Ref VPC + PublicSubnetIds: + Description: Subnet IDs of the public subnets + Value: + !Join [ + ",", + [!Ref PublicSubnet, !If [DoAz2, !Ref PublicSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PublicSubnet3, !Ref "AWS::NoValue"]] + ] + PrivateSubnetIds: + Description: Subnet IDs of the private subnets + Value: + !Join [ + ",", + [!Ref PrivateSubnet, !If [DoAz2, !Ref PrivateSubnet2, !Ref "AWS::NoValue"], !If [DoAz3, !Ref PrivateSubnet3, !Ref "AWS::NoValue"]] + ] +---- diff --git a/modules/installation-cloudformation-worker.adoc b/modules/installation-cloudformation-worker.adoc new file mode 100644 index 000000000000..36c90413d6d0 --- /dev/null +++ b/modules/installation-cloudformation-worker.adoc @@ -0,0 +1,127 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-upi.adoc + +[id="installation-cloudformation-worker-{context}"] += CloudFormation template for worker machines + +You can use the following CloudFormation template to deploy the worker machines +that you need for your {product-title} cluster. + +[source,yaml] +---- +AWSTemplateFormatVersion: 2010-09-09 +Description: Template for Openshift Cluster UPI Node Launch (EC2 worker instance) + +Parameters: + InfrastructureName: + AllowedPattern: ^([a-zA-Z][a-zA-Z0-9\-]{0,31})$ + MaxLength: 32 + MinLength: 1 + ConstraintDescription: Infrastructure name must be alphanumeric, start with a letter and a maximum of 32 characters + Description: A short, unique cluster ID used to tag nodes for the kubelet cloud provider. + Type: String + RhcosAmi: + Description: Current RHEL CoreOS AMI to use for boostrap + Type: AWS::EC2::Image::Id + WorkerSubnet: + Description: The subnets (recommend private) to launch the master nodes into + Type: AWS::EC2::Subnet::Id + WorkerSecurityGroupId: + Description: The master security group ID to associate with master nodes. + Type: AWS::EC2::SecurityGroup::Id + IgnitionLocation: + Default: https://api-int.$CLUSTER_NAME.$DOMAIN:22623/config/worker + Description: Location to fetch bootstrap ignition from. (Recommend to use the autocreated ignition config location.) + Type: String + CertificateAuthorities: + Default: data:text/plain;charset=utf-8;base64,ABC...xYz== + Description: Base64 encoded certificate authority string to use. + Type: String + WorkerInstanceProfileName: + Description: IAM profile to associate with master nodes. + Type: String + WorkerInstanceType: + Default: m4.large + Type: String + AllowedValues: + - "m4.large" + - "m4.xlarge" + - "m4.2xlarge" + - "m4.4xlarge" + - "m4.8xlarge" + - "m4.10xlarge" + - "m4.16xlarge" + - "c4.large" + - "c4.xlarge" + - "c4.2xlarge" + - "c4.4xlarge" + - "c4.8xlarge" + - "r4.large" + - "r4.xlarge" + - "r4.2xlarge" + - "r4.4xlarge" + - "r4.8xlarge" + - "r4.16xlarge" + +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Host Information" + Parameters: + - WorkerInstanceType + - RhcosAmi + - IgnitionLocation + - CertificateAuthorities + - WorkerSecurityGroupId + - WorkerInstanceProfileName + - Label: + default: "Network Configuration" + Parameters: + - WorkerSubnet + ParameterLabels: + WorkerSubnet: + default: "Worker Subnet" + WorkerInstanceType: + default: "Worker Instance Type" + WorkerInstanceProfileName: + default: "Worker Instance Profile Name" + RhcosAmi: + default: "RHEL CoreOS AMI ID" + IgnitionLocation: + default: "Worker Ignition Source" + CertificateAuthorities: + default: "Ignition CA String" + WorkerSecurityGroupId: + default: "Worker Security Group ID" + +Resources: + Worker0: + Type: AWS::EC2::Instance + Properties: + ImageId: !Ref RhcosAmi + IamInstanceProfile: !Ref WorkerInstanceProfileName + InstanceType: !Ref WorkerInstanceType + NetworkInterfaces: + - AssociatePublicIpAddress: "false" + DeviceIndex: "0" + GroupSet: + - !Ref "WorkerSecurityGroupId" + SubnetId: !Ref "WorkerSubnet" + UserData: + Fn::Base64: !Sub + - '{"ignition":{"config":{"append":[{"source":"${SOURCE}","verification":{}}]},"security":{"tls":{"certificateAuthorities":[{"source":"${CA_BUNDLE}","verification":{}}]}},"timeouts":{},"version":"2.2.0"},"networkd":{},"passwd":{},"storage":{},"systemd":{}}' + - { + SOURCE: !Ref IgnitionLocation, + CA_BUNDLE: !Ref CertificateAuthorities, + } + Tags: + - Key: !Join ["", ["kubernetes.io/cluster/", !Ref InfrastructureName]] + Value: "shared" + +Outputs: + PrivateIP: + Description: The compute node private IP address + Value: !GetAtt Worker0.PrivateIp +---- \ No newline at end of file diff --git a/modules/installation-configuration-parameters.adoc b/modules/installation-configuration-parameters.adoc index 40bc01b9e48e..524bb165a171 100644 --- a/modules/installation-configuration-parameters.adoc +++ b/modules/installation-configuration-parameters.adoc @@ -81,11 +81,11 @@ link:https://cloud.openshift.com/clusters/install[the OpenShift start page]. |`compute.platform.aws.rootVolume.iops` |The Input/Output Operations Per Second (IOPS) that is reserved for the root volume. -|Integer, for example `4000` +|Integer, for example `4000`. |`compute.platform.aws.rootVolume.size` |The size in GiB of the root volume. -|Integer, for example `500` +|Integer, for example `500`. |`compute.platform.aws.rootVolume.type` |The instance type of the root volume. @@ -106,7 +106,7 @@ link:https://yaml.org/spec/1.2/spec.html#sequence//[YAML sequence]. |`compute.aws.region` |The AWS region that the installation program creates compute resources in. |Valid link:https://docs.aws.amazon.com/general/latest/gr/rande.html[AWS region], -such as `us-east-1` +such as `us-east-1`. |`controlPlane.platform.aws.type` |The EC2 instance type for the control plane machines. @@ -122,7 +122,7 @@ link:https://yaml.org/spec/1.2/spec.html#sequence//[YAML sequence]. |`controlPlane.aws.region` |The AWS region that the installation program creates control plane resources in. |Valid link:https://docs.aws.amazon.com/general/latest/gr/rande.html[AWS region], -such as `us-east-1` +such as `us-east-1`. |`platform.aws.userTags` |A map of keys and values that the installation program adds as tags to all diff --git a/modules/installation-creating-aws-bootstrap.adoc b/modules/installation-creating-aws-bootstrap.adoc new file mode 100644 index 000000000000..6c7c5bc0984c --- /dev/null +++ b/modules/installation-creating-aws-bootstrap.adoc @@ -0,0 +1,203 @@ +// Module included in the following assemblies: +// +// * installing/installing_aws_upi/installing-aws-user-infra.adoc + +[id="installation-creating-aws-bootstrap-{context}"] += Creating the bootstrap node in AWS + +You must create the bootstrap node in Amazon Web Services (AWS) to use during +{product-title} cluster initialization. The easiest way to create this node is +to modify the provided CloudFormation template. + +[NOTE] +==== +If you do not use the provided CloudFormation template to create your bootstrap +node, you must review the provided information and manually create +the infrastructure. If your cluster does not initialize correctly, you might +have to contact Red Hat support with your installation logs. +==== + +.Prerequisites + +* Configure an AWS account. +* Generate the Ignition config files for your cluster. +* Create and configure a VPC and assocated subnets in AWS. +* Create and configure DNS, load balancers, and listeners in AWS. +* Create control plane and compute roles. + +.Procedure + +. Provide a location to serve the `bootstrap.ign` Ignition config file to your +cluster. This file is located in your installation directory. One way to do this +is to create an S3 bucket in your cluster's region and upload the Ingition +config file to it. ++ +[IMPORTANT] +==== +The provided CloudFormation Template assumes that the +Ignition config files for your cluster are served from an S3 bucket. If you +choose to serve the files from another location, you must modify the templates. +==== ++ +[NOTE] +==== +The bootstrap Ignition config file does contain secrets, like X.509 keys. The +following steps provide basic security for the S3 bucket. To provide additional +security, you can enable an S3 bucket policy to allow only certain users, such +as the OpenShift IAM user, to access objects that the bucket contains. You +can avoid S3 entirely and serve your bootstrap Ignition config file from any +address that the bootstrap machine can reach. +==== + +.. Create the bucket: ++ +---- +$ aws s3 mb s3://-upi <1> +---- +<1> `-upi` is the bucket name. + +.. Upload the `bootstrap.ign` Ignition config file to the bucket: ++ +---- +$ aws s3 cp bootstrap.ign s3://-upi/bootstrap.ign +---- + +.. Verify that the file uploaded: ++ +---- +$ aws s3 ls s3://-upi/ + +2019-04-03 16:15:16 314878 bootstrap.ign +---- + +. Create a JSON file that contains the parameter values that the template +requires: ++ +[source,json] +---- +[ + { + "ParameterKey": "InfrastructureName", <1> + "ParameterValue": "mycluster-vw9j6" <2> + }, + { + "ParameterKey": "RhcosAmi", <3> + "ParameterValue": "ami-07f604c5f18a1e7a9" <4> + }, + { + "ParameterKey": "AllowedBootstrapSshCidr", <5> + "ParameterValue": "0.0.0.0/0" <6> + }, + { + "ParameterKey": "PublicSubnet", <7> + "ParameterValue": "subnet-09c2adf329727bc59" <8> + }, + { + "ParameterKey": "MasterSecurityGroupId", <9> + "ParameterValue": "sg-0a53cd3337c15a2ee" <10> + }, + { + "ParameterKey": "VpcId", <11> + "ParameterValue": "vpc-055h956b1e6572be9" <12> + }, + { + "ParameterKey": "BootstrapIgnitionLocation", <13> + "ParameterValue": "s3:///bootstrap.ign" <14> + }, + { + "ParameterKey": "AutoRegisterELB", <15> + "ParameterValue": "yes" <16> + }, + { + "ParameterKey": "RegisterNlbIpTargetsLambdaArn", <17> + "ParameterValue": "arn:aws:lambda::269733483066:function:-RegisterNlbIpTargets-NHEKB9IHN5XZ" <18> + }, + { + "ParameterKey": "ExternalApiTargetGroupArn", <19> + "ParameterValue": "arn:aws:elasticloadbalancing::269733383066:targetgroup/-Exter-D374J8IQ3K03/b26760f600ff322e" <20> + }, + { + "ParameterKey": "InternalApiTargetGroupArn", <21> + "ParameterValue": "arn:aws:elasticloadbalancing::269733383066:targetgroup/-Inter-1Q3IAOSC8ZNNM/8e85701acf15632d" <22> + }, + { + "ParameterKey": "InternalServiceTargetGroupArn", <23> + "ParameterValue": "arn:aws:elasticloadbalancing::269733383066:targetgroup/-Inter-1INRIWR8L3627/4cd9e764bb418f47" <24> + } +] + +---- +<1> The name for your cluster infrastructure that is encoded in your Ignition +config files for the cluster. +<2> Specify the infrastructure name that you extracted from the Ignition config +file metadata, which has the format `-`. +<3> Current {op-system-first} AMI to use for the boostrap node. +<4> Specify a valid `AWS::EC2::Image::Id` value. +<5> CIDR block to allow SSH access to the bootstrap node. +<6> Specify a CIDR block in the format `x.x.x.x/16-24`. +<7> The public subnet that is associated with your VPC to launch the bootstrap +node into. +<8> Specify the `PublicSubnetIds` value from the output of the CloudFormation +template for the VPC. +<9> The master security group ID (for registering temporary rules) +<10> Specify the `MasterSecurityGroupId` value from the output of the +CloudFormation template for the security group and roles. +<11> The VPC created resources will belong to. +<12> Specify the `VpcId` value from the output of the CloudFormation template +for the VPC. +<13> Location to fetch bootstrap Ignition config file from. +<14> Specify the S3 bucket and file name in the form +`s3:///bootstrap.ign`. +<15> Whether or not to register a network load balancer (NLB). +<16> Specify `yes` or `no`. If you specify `yes`, you must provide a Lambda +Amazon Resource Name (ARN) value. +<17> The ARN for NLB IP target registration lambda group. +<18> Specify the `RegisterNlbIpTargetsLambda` value from the output of the +CloudFormation template for DNS and load balancing. +<19> The ARN for external API load balancer target group. +<20> Specify the `ExternalApiTargetGroupArn` value from the output of the +CloudFormation template for DNS and load balancing. +<21> The ARN for internal API load balancer target group. +<22> Specify the `InternalApiTargetGroupArn` value from the output of the +CloudFormation template for DNS and load balancing. +<23> The ARN for internal service load balancer target group. +<24> Specify the `InternalServiceTargetGroupArn` value from the output of the +CloudFormation template for DNS and load balancing. + +. Copy the template from the *CloudFormation template for the bootstrap machine* +section of this topic and save it as a YAML file on your computer. This template +describes the bootstrap machine that your cluster requires. + +. Launch the template: ++ +[IMPORTANT] +==== +You must enter the command on a single line. +==== ++ +---- +$ aws cloudformation create-stack --stack-name <1> + --template-body file://