diff --git a/cf-deploy-akto b/cf-deploy-akto index 771dbc6..214e88c 100755 --- a/cf-deploy-akto +++ b/cf-deploy-akto @@ -105,16 +105,16 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then fi else log_step 'downloading...' - curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/usage-infra.zip + curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/prometheus-setup-usage-infra.zip log_step 'unpacking...' unzip master.zip - mv infra-feature-usage-infra infra + mv infra-feature-prometheus-setup-usage-infra infra fi # NB: this is to make onprem containers to all get named the same. cd infra if ! command_present unzip; then - git checkout feature/usage-infra + git checkout feature/prometheus-setup-usage-infra fi DOCKER_CONTEXT="$(pwd)" diff --git a/docker-compose-usage.yml b/docker-compose-usage.yml index 0ff2df3..abc6292 100644 --- a/docker-compose-usage.yml +++ b/docker-compose-usage.yml @@ -50,9 +50,37 @@ services: - ./data-kafka-secrets:/etc/kafka/secrets depends_on: - zoo1 - + + prometheus-pushgateway: + image: prom/pushgateway:latest + ports: + - "9091:9091" + container_name: prometheus-pushgateway + + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + restart: unless-stopped + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prom-data:/prometheus + depends_on: + - "prometheus-pushgateway" + container_name: prometheus + + grafana: + image: grafana/grafana:latest + volumes: + - grafana-storage:/var/lib/grafana + ports: + - "3000:3000" + depends_on: + - "prometheus" + container_name: grafana + usage-collector: - image: aktosecurity/usage-collector:latest + image: aktosecurity/usage-collector:m-917-merge restart: on-failure:10 ports: - "5000:5000" @@ -60,8 +88,12 @@ services: - kafka1 usage-processor: - image: aktosecurity/usage-processor:latest + image: aktosecurity/usage-processor:m-917-merge restart: on-failure:10 env_file: ./docker-usage.env depends_on: - kafka1 + +volumes: + grafana-storage: + prom-data: \ No newline at end of file diff --git a/prometheus.yml b/prometheus.yml new file mode 100644 index 0000000..9657cd1 --- /dev/null +++ b/prometheus.yml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['prometheus:9090'] + + - job_name: 'pushgateway' + scrape_interval: 5s + honor_labels: true + static_configs: + - targets: ['prometheus-pushgateway:9091'] diff --git a/templates/akto-quick-setup.yaml b/templates/akto-quick-setup.yaml index 0b274ff..076ccd1 100644 --- a/templates/akto-quick-setup.yaml +++ b/templates/akto-quick-setup.yaml @@ -224,10 +224,10 @@ Resources: - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - export COMPOSE_FILE=docker-compose-mongo.yml - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -339,36 +339,24 @@ Resources: Fn::Join: - "\n" - - "#!/bin/bash -xe" + - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - touch /tmp/hello.txt - touch ~/hello.txt - - Fn::Sub: export AKTO_MONGO_CONN='${AktoMongoInstance.PrivateIp}' - sudo yum update -y - sudo yum install -y python python-setuptools - sudo yum install -y docker - sudo dockerd& - sudo mkdir -p /opt/aws/bin - - export COMPOSE_FILE=docker-compose-dashboard.yml - sudo wget https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz - sudo python -m easy_install --script-dir /opt/aws/bin aws-cfn-bootstrap-latest.tar.gz - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto' > cf-deploy-akto + - export COMPOSE_FILE=docker-compose-usage.yml + - sudo systemctl enable /usr/lib/systemd/system/docker.service + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto' > cf-deploy-akto - sudo chmod 700 cf-deploy-akto - "./cf-deploy-akto < <(echo 'test')" - - sudo echo >> ~/akto/infra/docker-dashboard.env - - sudo echo AKTO_MONGO_CONN=mongodb://$AKTO_MONGO_CONN:27017/admini >> - ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_KEY_PAIR='${KeyPair}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo OWNER_EMAIL='${UserEmail}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo EC2_SUBNET_ID='${PrivateSubnetId}' >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_ACCOUNT_ID=${AWS::AccountId} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AWS_DEFAULT_REGION=${AWS::Region} >> ~/akto/infra/docker-dashboard.env - - Fn::Sub: sudo echo AKTO_DASHBOARD_STACK_NAME=${AWS::StackName} >> ~/akto/infra/docker-dashboard.env - - Fn::Join: - - "" - - - "sudo echo AKTO_DASHBOARD_ROLE_NAME=" - - !Ref AktoDashboardRole - - " >> ~/akto/infra/docker-dashboard.env" - - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/quick-setup/cf-deploy-akto-start' > cf-deploy-akto-start + - sudo echo >> ~/akto/infra/docker-billing.env + - sudo echo DB_CONN_URL=mongodb://$AKTO_MONGO_CONN:27017/admini >> ~/akto/infra/docker-usage.env + - curl -fsSL 'https://raw.githubusercontent.com/akto-api-security/infra/feature/prometheus-setup-usage-infra/cf-deploy-akto-start' > cf-deploy-akto-start - sudo chmod 700 cf-deploy-akto-start - "./cf-deploy-akto-start < <(echo 'test')" - Fn::Join: @@ -415,12 +403,12 @@ Resources: AktoDashboardTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: - Port: "8080" + Port: "5000" Protocol: HTTP HealthCheckEnabled: "true" HealthCheckIntervalSeconds: 10 - HealthCheckPath: "/metrics" - HealthCheckPort: "8080" + HealthCheckPath: "/" + HealthCheckPort: "5000" HealthCheckProtocol: HTTP HealthCheckTimeoutSeconds: 6 HealthyThresholdCount: 2