collect-application-log.yml

ROSTemplateFormatVersion: '2015-09-01'
Description:
  zh-cn: 创建VPC、ECS实例，模拟生成日志，配置SLS项目、存储库、索引及采集配置，实现日志监控与查询。
  en: Create a VPC (Virtual Private Cloud), provision ECS (Elastic Compute Service)
    instances, simulate log generation, configure SLS (Log Service) projects, repositories,
    indices, and collection configurations to facilitate log monitoring and querying
    capabilities.
Parameters:
  CommonName:
    Type: String
    Default: collect-app-log
  SlsProjectName:
    Type: String
    Label:
      en: Name of sls project
      zh-cn: 日志项目的名称
    Description:
      en: >-
        The name contains 3 to 36 characters. It must start and end with 
        a lowercase letter or number. The value can contain lowercase 
        letters, digits, and hyphens (-).
      zh-cn: 长度为3~36个字符。必须以小写英文字母或数字开头和结尾。可包含小写英文字母、数字和短划线（-）。
    AssociationProperty: AutoCompleteInput
    AssociationPropertyMetadata:
      Length: 5
      Prefix: biz-log-test-
      CharacterClasses:
        - Class: lowercase
          min: 1
  ZoneId:
    Type: String
    AssociationProperty: 'ALIYUN::ECS::Instance::ZoneId'
    AssociationPropertyMetadata:
      AutoSelectFirst: true
    Label:
      en: Availability Zone
      zh-cn: 可用区
  InstanceType:
    Type: String
    AssociationProperty: 'ALIYUN::ECS::Instance::InstanceType'
    AssociationPropertyMetadata:
      SpotStrategy: SpotAsPriceGo
      InstanceChargeType: PostPaid
      SystemDiskCategory: cloud_essd
      ZoneId: ${ZoneId}
    Label:
      en: Instance Type
      zh-cn: 实例规格
    Description:
      zh-cn: 本方案会创建一个抢占式实例，模拟日志采集。
      en: >-
        This solution will create a spot instance and simulate log collection.
    Default: ecs.c7.large
  InstancePassword:
    NoEcho: true
    Type: String
    Description:
      en: >-
        Server login password, Length 8-30, must contain three(Capital letters,
        lowercase letters, numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special
        symbol in)
      zh-cn: >-
        服务器登录密码，长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/
        中的特殊符号）
    Label:
      en: Instance Password
      zh-cn: 实例密码
    ConstraintDescription:
      en: >-
        Length 8-30, must contain three(Capital letters, lowercase letters,
        numbers, ()`~!@#$%^&*_-+=|{}[]:;'<>,.?/ Special symbol in)
      zh-cn: '长度8-30，必须包含三项（大写字母、小写字母、数字、 ()`~!@#$%^&*_-+=|{}[]:;''<>,.?/ 中的特殊符号）'
    AssociationProperty: 'ALIYUN::ECS::Instance::Password'
    Default: null
Resources:
  Vpc:
    Type: 'ALIYUN::ECS::VPC'
    Properties:
      CidrBlock: 192.168.0.0/16
      VpcName:
        Fn::Sub: ${CommonName}-vpc
  VSwitch:
    Type: 'ALIYUN::ECS::VSwitch'
    Properties:
      VpcId:
        Ref: Vpc
      CidrBlock: 192.168.0.0/24
      ZoneId:
        Ref: ZoneId
      VSwitchName:
        Fn::Sub: ${CommonName}-vsw
  SecurityGroup:
    Type: 'ALIYUN::ECS::SecurityGroup'
    Properties:
      VpcId:
        Ref: Vpc
      SecurityGroupName:
        Fn::Sub: ${CommonName}-sg
      SecurityGroupIngress:
        - PortRange: 22/22
          SourceCidrIp: 0.0.0.0/0
          IpProtocol: tcp
        - PortRange: 443/443
          SourceCidrIp: 0.0.0.0/0
          IpProtocol: tcp
        - PortRange: 80/80
          SourceCidrIp: 0.0.0.0/0
          IpProtocol: tcp
        - PortRange: 3306/3306
          SourceCidrIp: 0.0.0.0/0
          IpProtocol: tcp
  EcsInstance:
    Type: 'ALIYUN::ECS::InstanceGroup'
    Properties:
      VpcId:
        Ref: Vpc
      ZoneId:
        Ref: ZoneId
      VSwitchId:
        Ref: VSwitch
      SecurityGroupId:
        Ref: SecurityGroup
      ImageId: aliyun_3_9_x64_20G_alibase_
      InstanceName:
        Fn::Sub: ${CommonName}-ecs
      InstanceType:
        Ref: InstanceType
      SystemDiskCategory: cloud_essd
      MaxAmount: 2
      InternetMaxBandwidthOut: 100
      SpotStrategy: SpotAsPriceGo
      Password:
        Ref: InstancePassword
  GenerateLog:
    Type: 'ALIYUN::ECS::RunCommand'
    Properties:
      InstanceIds:
        Fn::GetAtt:
          - EcsInstance
          - InstanceIds
      Type: RunShellScript
      Sync: true
      Timeout: 3600
      CommandContent:
        Fn::Sub: |-
          #!/bin/bash
          export PATH=/usr/local/bin:$PATH

          cat << EOF > sls-monitor-test.py
          # -*- coding: utf-8 -*-
          import sys
          import random
          import time
          import os
          from argparse import ArgumentParser
          from ipaddress import IPv4Address, IPv4Network
          
          MIN_SLEEP_RANGE = 1000
          MAX_SLEEP_RANGE = 5000
          IPV4_PUBLIC_NETWORKS = [
              IPv4Network('10.0.0.0/8'),
              IPv4Network('172.16.0.0/12'),
              IPv4Network('192.168.0.0/16'),
              IPv4Network('127.0.0.0/8'),
              IPv4Network('169.254.0.0/16')
          ]
          
          def generate_public_ipv4_address():
              while True:
                  ip_address = IPv4Address(random.randint(0, 2**32 - 1))
                  if not any(ip_address in network for network in IPV4_PUBLIC_NETWORKS):
                      return str(ip_address)
          
          def parse_args():
              parser = ArgumentParser()
              parser.add_argument('--output-file', required=True, help='Output file path')
              parser.add_argument('--min-sleep', type=int, default=MIN_SLEEP_RANGE,
                                  help='Minimum sleep interval in milliseconds (default: %d)' % MIN_SLEEP_RANGE)
              parser.add_argument('--max-sleep', type=int, default=MAX_SLEEP_RANGE,
                                  help='Maximum sleep interval in milliseconds (default: %d)' % MAX_SLEEP_RANGE)
              parser.add_argument('--failed-probability', type=float, default=0.1,
                                  help='Probability of generating a "failed" entry (default: %.1f)' % 0.1)
              parser.add_argument('--max-entries', type=int, default=sys.maxsize,
                                  help='Maximum number of entries to output (default: no limit)')
              return parser.parse_args()
          
          def write_to_file(message, log_level, userid, code, client_ip):
              with open(args.output_file, mode='a', encoding='utf-8') as f:
                  f.write(f'level: {log_level} userid: {userid} action: login code: {code} client_ip: {client_ip}\n')
          
          def main():
              global args
              args = parse_args()
              entry_count = 0
          
              while entry_count < args.max_entries:
                  # Generate random user ID
                  userid = random.randint(10000, 10000000)
                  # Generate random IP address
                  client_ip = generate_public_ipv4_address()
                  # Determine whether to log a successful or failed event
                  if random.random() < args.failed_probability:
                      code = 'failed'
                      log_level = 'WARNING'
                  else:
                      code = 'success'
                      log_level = 'INFO'
                  message = None
                  write_to_file(message, log_level, userid, code, client_ip)
                  entry_count += 1
          
                  # Sleep for a random interval within the specified range
                  sleep_time = random.randint(args.min_sleep, args.max_sleep) / 1000.0
                  time.sleep(sleep_time)
          
          if __name__ == '__main__':
              main()
          EOF
          nohup python3 sls-monitor-test.py  --output-file=/tmp/sls-monitor-test.log --max-sleep=2000 --min-sleep=500 --failed-probability=0.1 --max-entries=100000 > output.log 2>&1 &
  AutoEnableSLS:
    Type: ALIYUN::ROS::AutoEnableService
    Properties:
      ServiceName: SLS
  SlsProject:
    DependsOn: AutoEnableSLS
    Type: ALIYUN::SLS::Project
    Properties:
      Name:
        Ref: SlsProjectName
  SlsLogStore:
    Type: ALIYUN::SLS::Logstore
    Properties:
      LogstoreName: biz-log-test
      ProjectName:
        Ref: SlsProject
  SlsIndex:
    Type: ALIYUN::SLS::Index
    Properties:
      ProjectName:
        Ref: SlsProject
      FullTextIndex:
        Enable: true
      KeyIndices:
        - Name: content
          Type: text
      LogstoreName:
        Fn::GetAtt:
          - SlsLogStore
          - LogstoreName
  InstallLogAgent:
    Type: ALIYUN::OOS::Execution
    Properties:
      TemplateName: ACS-ECS-BulkyInstallLogAgent
      SafetyCheck: Skip
      Parameters:
        regionId:
          Ref: ALIYUN::Region
        OOSAssumeRole: ''
        targets:
          ResourceIds:
            Fn::GetAtt:
              - EcsInstance
              - InstanceIds
          RegionId:
            Ref: ALIYUN::Region
          Type: ResourceIds
        rateControl:
          Mode: Concurrency
          MaxErrors: 100%
          Concurrency: 1
        action: install
    DependsOn:
      - SlsIndex
  SlsMachineGroup:
    Type: ALIYUN::SLS::MachineGroup
    Properties:
      GroupName: sls-minitor-test
      ProjectName:
        Ref: SlsProject
      MachineIdentifyType: ip
      MachineList:
        Fn::GetAtt:
          - EcsInstance
          - PrivateIps
    DependsOn:
      - InstallLogAgent
  SlsLogtailConfig:
    Type: ALIYUN::SLS::LogtailConfig
    Properties:
      ProjectName:
        Ref: SlsProject
      LogtailConfigName: sls-monitor-test
      LogstoreName:
        Fn::GetAtt:
          - SlsLogStore
          - LogstoreName
      RawConfigData:
        configName: sls-monitor-test
        inputDetail:
          adjustTimezone: false
          delayAlarmBytes: 0
          delaySkipBytes: 0
          discardNonUtf8: false
          discardUnmatch: true
          dockerFile: false
          enableRawLog: false
          fileEncoding: utf8
          filePattern: sls-monitor-test.log
          filterKey: [ ]
          filterRegex: [ ]
          key:
            - content
          localStorage: true
          logBeginRegex: .*
          logPath: /tmp
          logTimezone: ''
          logType: common_reg_log
          maxDepth: 0
          maxSendRate: -1
          mergeType: topic
          preserve: true
          preserveDepth: 0
          priority: 0
          regex: (.*)
          sendRateExpire: 0
          sensitive_keys: [ ]
          tailExisted: false
          timeFormat: ''
          topicFormat: none
        inputType: file
        logSample: ''
        outputDetail:
          logstoreName:
            Fn::GetAtt:
              - SlsLogStore
              - LogstoreName
        outputType: LogService
  ApplyConfigToMachineGroup:
    Type: ALIYUN::SLS::ApplyConfigToMachineGroup
    Properties:
      ProjectName:
        Ref: SlsProject
      ConfigName:
        Fn::GetAtt:
          - SlsLogtailConfig
          - LogtailConfigName
      GroupName:
        Fn::GetAtt:
          - SlsMachineGroup
          - GroupName
Outputs:
  EcsLoginAddress:
    Description:
      zh-cn: ECS登录地址。
      en: Ecs login address.
    Value:
      'Fn::Sub': https://ecs-workbench.aliyun.com/?from=EcsConsole&instanceType=ecs&regionId=${ALIYUN::Region}&instanceId=${EcsInstance}
  LogQueryAddress:
    Description:
      zh-cn: 日志查询地址。
      en: Log query address.
    Value:
      'Fn::Sub': https://sls.console.aliyun.com/lognext/project/${SlsProject}/logsearch/${SlsLogStore.LogstoreName}
Metadata:
  'ALIYUN::ROS::Interface':
    ParameterGroups:
      - Parameters:
          - InstanceType
          - InstancePassword
          - SlsProjectName
    TemplateTags:
      - acs:technical-solution:storage:应用业务日志监控与告警-tech_solu_118
    Hidden:
      - CommonName