forked from aliyun/ros-templates
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcloud-native-enterprise-data-lake.yml
223 lines (223 loc) · 7.75 KB
/
cloud-native-enterprise-data-lake.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
ROSTemplateFormatVersion: '2015-09-01'
Description:
en: 'This solution uses the big data analysis scenario as an example. The technical architecture of the solution includes the following infrastructure and cloud services: One EMR cluster: provides the big data computing engine; one object storage OSS: provides the big data storage. '
zh-cn: 本方案以大数据分析场景为例。本方案的技术架构包括以下基础设施和云服务：1个EMR集群：提供大数据计算引擎、1个对象存储OSS：提供大数据存储。
Parameters:
CommonName:
Type: String
Default: data-analysis
ZoneId:
Type: String
Label:
en: Availability Zone
zh-cn: 可用区
AssociationProperty: ZoneId
EcsInstanceType:
Type: String
Label:
en: ECS instance type.
zh-cn: ECS实例规格
AssociationProperty: ALIYUN::ECS::Instance::InstanceType
AssociationPropertyMetadata:
ZoneId: ${ZoneId}
SystemDiskCategory:
Type: String
Label:
en: System Disk Type
zh-cn: 系统盘类型
Description:
en: '<font color=''blue''><b>Optional values:</b></font><br>[cloud_efficiency: <font color=''green''>Efficient Cloud Disk</font>]<br>[cloud_ssd: <font color=''green''>SSD Cloud Disk</font>]<br>[cloud_essd: <font color=''green''>ESSD Cloud Disk</font>]<br>[cloud: <font color=''green''>Cloud Disk</font>]<br>[ephemeral_ssd: <font color=''green''>Local SSD Cloud Disk</font>]'
zh-cn: '<font color=''blue''><b>可选值：</b></font><br>[cloud_efficiency: <font color=''green''>高效云盘</font>]<br>[cloud_ssd: <font color=''green''>SSD云盘</font>]<br>[cloud_essd: <font color=''green''>ESSD云盘</font>]<br>[cloud: <font color=''green''>普通云盘</font>]<br>[ephemeral_ssd: <font color=''green''>本地SSD盘</font>]'
AssociationProperty: ALIYUN::ECS::Disk::SystemDiskCategory
AssociationPropertyMetadata:
LocaleKey: DiskCategory
InstanceType: ${EcsInstanceType}
ZoneId: ${ZoneId}
Default: cloud_essd
MasterRootPassword:
Type: String
Label:
en: ECS Master Node Instance Password
zh-cn: ECS Master节点实例密码
Description:
en: Server login password, Length 8-30, must contain three(Capital letters, lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in).
zh-cn: 服务器登录密码,长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
ConstraintDescription:
en: Length 8-30, must contain three(Capital letters, lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in).
zh-cn: 长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
AllowedPattern: '[0-9A-Za-z\_\-\&:;''<>,=%`~!@#\(\)\$\^\*\+\|\{\}\[\]\.\?\/]+$'
MinLength: 8
MaxLength: 30
NoEcho: true
BucketName:
ConstraintDescription:
zh-cn: 3 到 63 个字符，不以连字符 (-) 开头和结尾，可以包含小写字母、数字和连字符 (-)
en: >-
3 to 63 characters, not beginning and ending with a hyphen (-), can
contain lowercase letters, Numbers and hyphens (-)
Description:
zh-cn: >-
3到63个字符，不以连字符(-)开头和结尾，可以包含小写字母、数字和连字符(-)；<br>备注：<font
color='blue'><b>需要保证整个网络的唯一性，已经存在的无法创建</b></font>
en: >-
3 to 63 characters, not beginning and ending with a hyphen (-), can
contain lowercase letters, Numbers and hyphens (-);<br>Note: <font
color='blue'><b>need whole network uniqueness, already existing can not
be created.</b></font>
Default: example-bucket-data-lake
MinLength: 3
Label:
zh-cn: 存储空间名称
en: Bucket Name
AllowedPattern: '^[a-z0-9]+[a-z0-9\-]*[a-z0-9]+$'
MaxLength: 63
Type: String
Resources:
Vpc:
Type: ALIYUN::ECS::VPC
Properties:
CidrBlock: 192.168.0.0/16
VpcName:
Fn::Sub: vpc-created-by-ros-${CommonName}
VSwitch:
Type: ALIYUN::ECS::VSwitch
Properties:
ZoneId:
Ref: ZoneId
VpcId:
Ref: Vpc
CidrBlock: 192.168.0.0/24
VSwitchName:
Fn::Sub: vsw-created-by-ros-${CommonName}
SecurityGroup:
Type: ALIYUN::ECS::SecurityGroup
Properties:
VpcId:
Ref: Vpc
SecurityGroupIngress_22:
Type: ALIYUN::ECS::SecurityGroupIngress
Properties:
SecurityGroupId:
Ref: SecurityGroup
SourceCidrIp: 0.0.0.0/0
IpProtocol: tcp
NicType: intranet
PortRange: 22/22
SecurityGroupIngress_80:
Type: ALIYUN::ECS::SecurityGroupIngress
Properties:
SecurityGroupId:
Ref: SecurityGroup
SourceCidrIp: 0.0.0.0/0
IpProtocol: tcp
NicType: intranet
PortRange: 80/80
SecurityGroupIngress_3306:
Type: ALIYUN::ECS::SecurityGroupIngress
Properties:
SecurityGroupId:
Ref: SecurityGroup
SourceCidrIp: 0.0.0.0/0
IpProtocol: tcp
NicType: intranet
PortRange: 3306/3306
SecurityGroupIngress_443:
Type: ALIYUN::ECS::SecurityGroupIngress
Properties:
SecurityGroupId:
Ref: SecurityGroup
SourceCidrIp: 0.0.0.0/0
IpProtocol: tcp
NicType: intranet
PortRange: 443/443
Cluster:
Type: ALIYUN::EMR::Cluster2
Properties:
ApplicationConfigs:
- ConfigFileName: hivemetastore-site.xml
ApplicationName: HIVE
ConfigItemKey: hive.metastore.type
ConfigScope: CLUSTER
ConfigItemValue: DLF
- ConfigFileName: hivemetastore-site.xml
ApplicationName: SPARK2
ConfigItemKey: hive-site.xml
ConfigScope: CLUSTER
ConfigItemValue: DLF
Applications:
- ApplicationName: HADOOP-COMMON
- ApplicationName: HIVE
- ApplicationName: TEZ
- ApplicationName: YARN
- ApplicationName: HDFS
- ApplicationName: SPARK2
PaymentType: PayAsYouGo
ClusterType: DATALAKE
NodeGroups:
- NodeCount: 1
NodeGroupName: emr-master
DataDisks:
- Category: cloud_essd
Size: 80
Count: 1
VSwitchIds:
- Ref: VSwitch
SystemDisk:
Category:
Ref: SystemDiskCategory
Size: 80
NodeGroupType: MASTER
InstanceTypes:
- Ref: EcsInstanceType
- NodeCount: 2
NodeGroupName: emr-core
DataDisks:
- Category: cloud_essd
Size: 80
Count: 4
VSwitchIds:
- Ref: VSwitch
SystemDisk:
Category:
Ref: SystemDiskCategory
Size: 80
NodeGroupType: CORE
InstanceTypes:
- Ref: EcsInstanceType
ReleaseVersion: EMR-5.13.0
NodeAttributes:
VpcId:
Ref: Vpc
ZoneId:
Ref: ZoneId
SecurityGroupId:
Ref: SecurityGroup
MasterRootPassword:
Ref: MasterRootPassword
RamRole: AliyunECSInstanceForEMRRole
ClusterName:
Fn::Sub: Emr-DataLake-created-by-ros-${CommonName}
Bucket:
Type: ALIYUN::OSS::Bucket
Properties:
BucketName:
Ref: BucketName
DeletionForce: true
Metadata:
ALIYUN::ROS::Interface:
ParameterGroups:
- Parameters:
- ZoneId
- EcsInstanceType
- SystemDiskCategory
- MasterRootPassword
Label:
default: EMR
- Parameters:
- BucketName
Label:
default: OSS
TemplateTags:
- acs:technical-solution:data-analysis:云原生企业级数据湖
Hidden:
- CommonName