dts-cache-synchronization.yml

ROSTemplateFormatVersion: '2015-09-01'
Description:
  zh-cn: 该模版通过DTS实现MySQL与Redis的缓存同步，包括VPC、ECS、RDS、Redis实例创建及安全配置，同时设置消费组确保数据一致性。
  en: This template achieves cache synchronization between MySQL and Redis via DTS,
    encompassing the creation and security configuration of VPC, ECS, RDS, and Redis
    instances, alongside the establishment of consumer groups to ensure data consistency.
Parameters:
  ZoneId:
    Type: String
    Label:
      en: VSwitch Availability Zone
      zh-cn: 交换机可用区
    AssociationProperty: ALIYUN::ECS::Instance::ZoneId
  InstanceType:
    Type: String
    Label:
      en: Instance Type
      zh-cn: 实例类型
    Description:
      en: 'Fill in the specifications that can be used under the VSwitch availability
        zone;</b></font><br>general specifications：<font color=''red''><b>ecs.c5.large</b></font><br>note:
        a few zones do not support general specifications<br>see detail: <a href=''https://www.alibabacloud.com/help/en/doc-detail/25378.html''
        target=''_blank''><b><font color=''blue''>Instance Specification Family.</font></a></b>'
      zh-cn: 填写VSwitch可用区下可使用的规格；<br>通用规格：<font color='red'><b>ecs.c5.large</b></font><br>注：可用区可能不支持通用规格<br>规格详见：<a
        href='https://help.aliyun.com/document_detail/25378.html' target='_blank'><b><font
        color='blue'>实例规格族。</font></a></b>
    AssociationProperty: ALIYUN::ECS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
  SystemDiskCategory:
    Type: String
    Label:
      en: System Disk Type
      zh-cn: 系统盘类型
    Description:
      en: '<font color=''blue''><b>Optional values:</b></font><br>[cloud_efficiency: <font color=''green''>Efficient Cloud Disk</font>]<br>[cloud_ssd: <font color=''green''>SSD Cloud Disk</font>]<br>[cloud_essd: <font color=''green''>ESSD Cloud Disk</font>]<br>[cloud: <font color=''green''>Cloud Disk</font>]<br>[ephemeral_ssd: <font color=''green''>Local SSD Cloud Disk</font>]'
      zh-cn: '<font color=''blue''><b>可选值：</b></font><br>[cloud_efficiency: <font color=''green''>高效云盘</font>]<br>[cloud_ssd: <font color=''green''>SSD云盘</font>]<br>[cloud_essd: <font color=''green''>ESSD云盘</font>]<br>[cloud: <font color=''green''>普通云盘</font>]<br>[ephemeral_ssd: <font color=''green''>本地SSD盘</font>]'
    AssociationProperty: ALIYUN::ECS::Disk::SystemDiskCategory
    AssociationPropertyMetadata:
      LocaleKey: DiskCategory
      InstanceType: ${InstanceType}
      ZoneId: ${ZoneId}
  InstancePassword:
    Type: String
    Label:
      en: Instance Password
      zh-cn: 实例密码
    Description:
      en: Server login password, Length 8-30, must contain three(Capital letters,
        lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol
        in).
      zh-cn: 服务器登录密码,长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
    ConstraintDescription:
      en: Length 8-30, must contain three(Capital letters, lowercase letters, numbers,
        ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in).
      zh-cn: 长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ 中的特殊符号）。
    AllowedPattern: '[0-9A-Za-z\_\-\&:;''<>,=%`~!@#\(\)\$\^\*\+\|\{\}\[\]\.\?\/]+$'
    MinLength: 8
    MaxLength: 30
    NoEcho: true
  DBName:
    Type: String
    Label:
      en: DB Name
      zh-cn: 数据库名
    Description:
      en: Name of database.
      zh-cn: RDS数据库名。
    ConstraintDescription:
      en: Consist of 2 to 64 characters of lowercase letters, underline. Must begin
        with a letter and be end with an alphanumeric character.
      zh-cn: 由 2 到 64 个小写字母组成，下划线。必须以字母开头，以字母数字字符结尾。
    Default: demodb
    MinLength: 2
    MaxLength: 64
  RDSDBUser:
    Type: String
    Label:
      en: RDS DB Username
      zh-cn: RDS数据库账号
    Description:
      en: Username of database.
      zh-cn: RDS数据库账号。
    ConstraintDescription:
      en: Consist of 2 to 16 characters of lowercase letters, underline. Must begin
        with a letter and be end with an alphanumeric character.
      zh-cn: 由 2 到 16 个小写字母组成，下划线。必须以字母开头，以字母数字字符结尾。
    Default: demouser123
    MinLength: 2
    MaxLength: 16
  RDSDBPassword:
    Type: String
    Label:
      en: RDS DB Password
      zh-cn: RDS数据库密码
    Description:
      en: RDS database password, consisting of letters, numbers, and underline(_),
        is 8 to 32 characters long,Must contain three different types of characters.
      zh-cn: RDS数据库密码，由字母、数字、下划线（_）组成，长度为8~32个字符，必须包含3种不同类型的字符。
    ConstraintDescription:
      en: Consisting of letters, numbers, and underline(_),
        is 8 to 32 characters long.
      zh-cn: 由字母、数字、下划线（_）组成，长度为8~32个字符。
    AssociationProperty: ALIYUN::RDS::Instance::AccountPassword
    MinLength: 8
    MaxLength: 32
    NoEcho: true
  DBInstanceClass:
    Type: String
    Label:
      en: Instance Class
      zh-cn: 实例规格
    AssociationProperty: ALIYUN::RDS::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
      EngineVersion: "8.0"
      Engine: MySQL
      Category: HighAvailability
      DBInstanceStorageType: cloud_essd
      CommodityCode: bards
  RedisInstanceClass:
    Type: String
    Label:
      en: Instance Type
      zh-cn: 实例规格
    Description:
      en: Instance specifications, please refer to：<a href='https://www.alibabacloud.com/help/en/doc-detail/26350.html'
        target='_blank'><b><font color='blue'>View specifications</font></b></a>
      zh-cn: 推荐规格：<b><font color='blue'>redis.master.small.default</font></b><br>更多实例规格，请参考：<a
        href='https://help.aliyun.com/document_detail/26350.html' target='_blank'><b><font
        color='blue'>查看规格信息</font></b></a>
    AssociationProperty: ALIYUN::Redis::Instance::InstanceType
    AssociationPropertyMetadata:
      ZoneId: ${ZoneId}
  RedisPassword:
    Type: String
    Label:
      en: Instance Password
      zh-cn: 实例密码
    Description:
      en: 'Length 8-32 characters, can contain size letters, Numbers and special symbols,
        including:! @ # $ % ^ & * ( ) _ + - ='
      zh-cn: 长度8-32个字符,可包含大小字母、数字及特殊符号（包含：!@#$%^&*()_+-=）
    ConstraintDescription:
      en: '8-32 characters, can contain size letters, Numbers and special symbols,
        including:! @ # $ % ^ & * ( ) _ + - ='
      zh-cn: 8-32个字符,可包含大小字母、数字及特殊符号（包含：!@#$%^&*()_+-=）
    MinLength: '8'
    MaxLength: '32'
    NoEcho: true
  ConsumerGroupUserName:
    Type: String
    Label:
      en: Consumer Group User Name
      zh-cn: 消费组账号
    Description:
      en: The maximum length is 16 characters.  The value can contain letters, digits, and underscores (_).
      zh-cn: 最长16个字符。可包含英文字母、数字和下划线（_）。
    Default: demoUser
  ConsumerGroupPassword:
    Type: String
    Label:
      en: Consumer Group Password
      zh-cn: 消费组账号的密码
    Description:
      en: The name contains 8 to 32 characters.  Contains at least two types of letters, digits, and special characters.
      zh-cn: 长度为8~32个字符。至少包含英文字母、数字和特殊字符中的两种。
    MinLength: '8'
    MaxLength: '32'
    NoEcho: true
Resources:
  Vpc:
    Type: ALIYUN::ECS::VPC
    DependsOn: AliyunDTSDefaultRole
    Properties:
      CidrBlock: 192.168.0.0/16
  VSwitch:
    Type: ALIYUN::ECS::VSwitch
    Properties:
      ZoneId:
        Ref: ZoneId
      VpcId:
        Ref: Vpc
      VSwitchName:
        Fn::Join:
          - '-'
          - - VSwitch
            - StackId
            - Ref: ALIYUN::StackId
      CidrBlock: 192.168.0.0/24
  SecurityGroup:
    Type: ALIYUN::ECS::SecurityGroup
    Properties:
      VpcId:
        Ref: Vpc
  SecurityGroupIngress_3306:
    Type: ALIYUN::ECS::SecurityGroupIngress
    Properties:
      SecurityGroupId:
        Ref: SecurityGroup
      SourceCidrIp: 0.0.0.0/0
      IpProtocol: tcp
      NicType: intranet
      PortRange: 3306/3306
  EcsInstance:
    Type: ALIYUN::ECS::Instance
    Properties:
      VpcId:
        Ref: Vpc
      VSwitchId:
        Ref: VSwitch
      SecurityGroupId:
        Ref: SecurityGroup
      ImageId: centos_7_8
      InstanceType:
        Ref: InstanceType
      SystemDiskCategory:
        Ref: SystemDiskCategory
      Password:
        Ref: InstancePassword
      IoOptimized: optimized
      InternetMaxBandwidthOut: 80
  RdsDBInstance:
    Type: ALIYUN::RDS::DBInstance
    Properties:
      ZoneId:
        Ref: ZoneId
      VpcId:
        Ref: Vpc
      VSwitchId:
        Ref: VSwitch
      DBInstanceClass:
        Ref: DBInstanceClass
      DBInstanceStorage: 100
      Engine: MySQL
      EngineVersion: "8.0"
      MasterUserPassword:
        Ref: RDSDBPassword
      MasterUserType: Super
      MasterUsername:
        Ref: RDSDBUser
      DBMappings:
        - CharacterSetName: utf8mb4
          DBName:
            Ref: DBName
      Category: HighAvailability
      DBInstanceStorageType: cloud_essd
      SecurityIPList: 0.0.0.0/0
      AllocatePublicConnection: true
      SlaveZoneIds:
        - Ref: ZoneId
  RandomString:
    Type: ALIYUN::RandomString
    Properties:
      length: 10
      character_classes:
        - class: lowercase
          min: 10
  RedisInstance:
    Type: ALIYUN::REDIS::Instance
    Properties:
      ZoneId:
        Ref: ZoneId
      VpcId:
        Ref: Vpc
      VSwitchId:
        Ref: VSwitch
      InstanceClass:
        Ref: RedisInstanceClass
      InstanceName: rds_mysql2redis_redis
      EngineVersion: "5.0"
      Connections:
        PublicConnection:
          ConnectionPort: 6379
          ConnectionString:
            Fn::Sub: rds-mysql2redis-${RandomString.value}
      Password:
        Ref: RedisPassword
  Whitelist:
    Type: ALIYUN::REDIS::Whitelist
    Properties:
      InstanceId:
        Ref: RedisInstance
      SecurityIps: 0.0.0.0/0
  SubscriptionInstance:
    DependsOn:
      - RdsDBInstance
    Type: ALIYUN::DTS::SubscriptionInstance
    Properties:
      SourceEndpointInstanceType: MySQL
      Configuration:
        DbList:
          Fn::Sub: |-
            {"${DBName}":{"name":"${DBName}","all":true}}
        SubscriptionDataType:
          DDL: false
          DML: true
        SubscriptionInstanceName: Subscription
        SourceEndpoint:
          InstanceType: RDS
          UserName:
            Ref: RDSDBUser
          Password:
            Ref: RDSDBPassword
          InstanceID:
            Ref: RdsDBInstance
        SubscriptionInstanceNetworkType: vpc
        SubscriptionInstance:
          VPCId:
            Ref: Vpc
          VSwitchId:
            Ref: VSwitch
  ConsumerGroup:
    Type: ALIYUN::DTS::ConsumerGroup
    Properties:
      ConsumerGroupPassword:
        Ref: ConsumerGroupPassword
      ConsumerGroupUserName:
        Ref: ConsumerGroupUserName
      ConsumerGroupName: mysql2redis_consumer_group
      SubscriptionInstanceId:
        Ref: SubscriptionInstance
  WaitConditionHandle:
    Type: ALIYUN::ROS::WaitConditionHandle
    Properties: { }
    DependsOn:
      - Whitelist
      - SecurityGroupIngress_3306
  WaitCondition:
    Type: ALIYUN::ROS::WaitCondition
    Properties:
      Handle:
        Ref: WaitConditionHandle
      Timeout: 7200
      Count: 1
  InstallDTSMySQL2Redis:
    Type: ALIYUN::ECS::RunCommand
    Properties:
      InstanceIds:
        - Ref: EcsInstance
      Type: RunShellScript
      Sync: true
      Timeout: 7200
      CommandContent:
        Fn::Sub: |-
          #!/bin/bash
          yum install -y java-1.8.0-openjdk-devel maven wget unzip
          echo 'JAVA_HOME=/usr/lib/jvm/java-1.8.0' >> /etc/profile
          echo 'PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile
          source /etc/profile
          wget https://database-plg.oss-cn-beijing.aliyuncs.com/dbbp/mysql2redis/subscribe-demo-master.zip
          unzip subscribe-demo-master.zip
          rm -rf subscribe-demo-master.zip
          cd subscribe-demo-master
          FilePath='src/main/java/com/alibaba/dts/demo/DTSMySQL2RedisDemo.java'
          sed -i "s/your broker url/${SubscriptionInstance.PublicHost}/" $FilePath
          sed -i "s/your dts topic/${SubscriptionInstance.SubscribeTopic}/" $FilePath
          sed -i "s/your sid/${ConsumerGroup.ConsumerGroupID}/" $FilePath
          sed -i "s/your user name/${ConsumerGroupUserName}/" $FilePath
          sed -i "s/your password/${RDSDBPassword}/" $FilePath
          timestamp=`date -d "$time" +%s`
          sed -i "s/start timestamp/$timestamp/" $FilePath
          sed -i "s/your redis url/${RedisInstance.PublicConnectionString}/" $FilePath
          sed -i "s/your redis password/${RedisPassword}/" $FilePath
          mvn clean install
          LOG_FILE="message.log"
          nohup java -jar target/subscribe-demo-1.0-SNAPSHOT-jar-with-dependencies.jar  >> $LOG_FILE 2>&1 &

          while [ ! -f "$LOG_FILE" ]; do
              echo "$(date '+%Y-%m-%d %H:%M:%S')  Wait for FILE $LOG_FILE to generate..." >> $LOG_FILE
              sleep 60
          done
          echo "$(date '+%Y-%m-%d %H:%M:%S')  Program packaging complete." >> $LOG_FILE
          
          if grep -q "Error" "$LOG_FILE"; then
              error_message=$(grep "Error" message.log | head -n 1)
              ${WaitConditionHandle.CurlCli} --data-binary '{"status": "FAILURE", "reason": "An error message is displayed: $error_message. For details, log in to the ECS instance, see /root/subscribe-demo-master/message.log"}'
          else
              ${WaitConditionHandle.CurlCli} --data-binary '{"status": "SUCCESS"}'
          fi
    DependsOn:
      - Whitelist
      - SecurityGroupIngress_3306
  AliyunDTSDefaultRole:
    Type: 'ALIYUN::RAM::Role'
    Properties:
      RoleName: AliyunDTSDefaultRole
      Description: DTS默认使用此角色来访问您在其他云产品中的资源。
      AssumeRolePolicyDocument:
        Version: '1'
        Statement:
          - Action: 'sts:AssumeRole'
            Effect: Allow
            Principal:
              Service:
                - dts.aliyuncs.com
      MaxSessionDuration: 3600
      IgnoreExisting: true
      DeletionForce: true
      PolicyAttachments:
        System:
          - AliyunDTSRolePolicy
Outputs:
  RdsInstanceInnerConnectionString:
    Description:
      en: RDS public connection string.
      zh-cn: RDS外网连接串。
    Value:
      Fn::GetAtt:
        - RdsDBInstance
        - PublicConnectionString
Metadata:
  ALIYUN::ROS::Interface:
    ParameterGroups:
      - Parameters:
          - ZoneId
          - InstanceType
          - SystemDiskCategory
          - InstancePassword
        Label:
          default: ECS
      - Parameters:
          - DBInstanceClass
          - DBName
          - RDSDBUser
          - RDSDBPassword
        Label:
          default: RDS
      - Parameters:
          - RedisInstanceClass
          - RedisPassword
        Label:
          default: Redis
      - Parameters:
          - ConsumerGroupUserName
          - ConsumerGroupPassword
        Label:
          default: Dts
    TemplateTags:
      - acs:technical-solution:database:实时同步RDS与Redis构建缓存一致性-tech_solu_21