Skip to content

Commit 3b18932

Browse files
SchnitzelSchnitzel
committed
more refactoring
1 parent 946d636 commit 3b18932

23 files changed

+410
-55
lines changed

‎.lagoon.logs-db-secrets.yaml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
apiVersion: v1
2+
kind: Template
3+
metadata:
4+
creationTimestamp: null
5+
name: lagoon-secret-environment-template
6+
parameters:
7+
- name: LOGSDB_ADMIN_PASSWORD
8+
description: admin password of logs-db
9+
generate: expression
10+
from: "[a-zA-Z0-9]{32}"
11+
- name: LOGSDB_KIBANASERVER_PASSWORD
12+
description: kibana password of logs-db
13+
generate: expression
14+
from: "[a-zA-Z0-9]{32}"
15+
- name: SAFE_BRANCH
16+
description: Which branch this belongs to, special chars replaced with dashes
17+
required: true
18+
- name: SAFE_PROJECT
19+
description: Which project this belongs to, special chars replaced with dashes
20+
required: true
21+
- name: BRANCH
22+
description: Which branch this belongs to, original value
23+
required: true
24+
- name: PROJECT
25+
description: Which project this belongs to, original value
26+
required: true
27+
- name: LAGOON_GIT_SHA
28+
description: git hash sha of the current deployment
29+
required: true
30+
- name: OPENSHIFT_PROJECT
31+
description: Name of the Project that this service is in
32+
required: true
33+
objects:
34+
- kind: Secret
35+
apiVersion: v1
36+
metadata:
37+
name: logs-db-admin-password
38+
stringData:
39+
LOGSDB_ADMIN_PASSWORD: ${LOGSDB_ADMIN_PASSWORD}
40+
- kind: Secret
41+
apiVersion: v1
42+
metadata:
43+
name: logs-db-kibanaserver-password
44+
stringData:
45+
LOGSDB_KIBANASERVER_PASSWORD: ${LOGSDB_KIBANASERVER_PASSWORD}

‎.lagoon.yml

+7
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ additional-yaml:
55
path: .lagoon.secrets.yaml
66
command: create
77
ignore_error: true
8+
9+
logs-db-secrets:
10+
path: .lagoon.logs-db-secrets.yaml
11+
command: create
12+
ignore_error: true
13+
14+
815
tasks:
916
post-rollout:
1017
- run:

‎docker-compose.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ services:
258258
user: '111111111'
259259
labels:
260260
lagoon.type: elasticsearch
261+
lagoon.template: services/logs-db/.lagoon.yml
261262
lagoon.image: amazeeiolagoon/logs-db:${SAFE_BRANCH:-master}
262263
logs-db-ui:
263264
image: ${IMAGE_REPO:-lagoon}/logs-db-ui
@@ -266,6 +267,7 @@ services:
266267
- "5601:5601"
267268
labels:
268269
lagoon.type: kibana
270+
lagoon.template: services/logs-db-ui/.lagoon.yml
269271
lagoon.image: amazeeiolagoon/logs-db-ui:${SAFE_BRANCH:-master}
270272
logs2logs-db:
271273
image: ${IMAGE_REPO:-lagoon}/logs2logs-db

‎images/elasticsearch/Dockerfile

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ARG IMAGE_REPO
22
FROM ${IMAGE_REPO:-lagoon}/commons as commons
3-
FROM docker.elastic.co/elasticsearch/elasticsearch-platinum:6.1.1
3+
FROM docker.elastic.co/elasticsearch/elasticsearch-platinum:6.2.4
44

55
MAINTAINER amazee.io
66
ENV LAGOON=elasticsearch
@@ -30,4 +30,8 @@ RUN echo "xpack.security.enabled: false" >> config/elasticsearch.yml
3030

3131
ENV ES_JAVA_OPTS "-Xms200m -Xmx200m"
3232

33-
ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash", "/usr/local/bin/docker-entrypoint.sh"]
33+
VOLUME [ "/usr/share/elasticsearch/data" ]
34+
35+
ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.bash"]
36+
37+
CMD ["/usr/local/bin/docker-entrypoint.sh"]

‎images/elasticsearch/docker-entrypoint.sh

+1-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,4 @@ else
1313
echo "Using service name: ${K8S_SVC_NAME}"
1414
# copy the pristine version to the one that can be edited
1515
/usr/bin/peer-finder -on-start="/lagoon/configure-es.sh" -service=${K8S_SVC_NAME}
16-
fi
17-
18-
exec $@
16+
fi

‎images/kibana/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ARG IMAGE_REPO
22
FROM ${IMAGE_REPO:-lagoon}/commons as commons
3-
FROM docker.elastic.co/kibana/kibana-x-pack:6.1.1
3+
FROM docker.elastic.co/kibana/kibana-x-pack:6.2.4
44

55
MAINTAINER amazee.io
66
ENV LAGOON=kibana

‎images/logstash/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
ARG IMAGE_REPO
22
FROM ${IMAGE_REPO:-lagoon}/commons as commons
3-
FROM docker.elastic.co/logstash/logstash:6.1.1
3+
FROM docker.elastic.co/logstash/logstash:6.2.4
44

55
MAINTAINER amazee.io
66
ENV LAGOON=logstash

‎services/api/src/dao/environment.js

+1-10
Original file line numberDiff line numberDiff line change
@@ -178,19 +178,10 @@ const getEnvironmentHitsMonthByEnvironmentId = ({ esClient }) => async (cred, op
178178
const month_leading_zero = interested_month.getMonth()+1 < 10 ? `0${interested_month.getMonth()+1}`: interested_month.getMonth()+1;
179179

180180
const result = await esClient.count({
181-
index: `router-logs-${interested_month.getFullYear()}.${month_leading_zero}.*`,
181+
index: `router-logs-${openshift_projectname}-${interested_month.getFullYear()}.${month_leading_zero}`,
182182
body: {
183183
"query": {
184184
"bool": {
185-
"must": [
186-
{
187-
"match_phrase": {
188-
"openshift_project": {
189-
"query": openshift_projectname
190-
}
191-
}
192-
}
193-
],
194185
"must_not": [
195186
{
196187
"match_phrase": {

‎services/logs-db-ui/.lagoon.yml

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
apiVersion: v1
2+
kind: Template
3+
metadata:
4+
creationTimestamp: null
5+
name: lagoon-openshift-template-kibana
6+
parameters:
7+
- name: SERVICE_NAME
8+
description: Name of this service
9+
required: true
10+
- name: SAFE_BRANCH
11+
description: Which branch this belongs to, special chars replaced with dashes
12+
required: true
13+
- name: SAFE_PROJECT
14+
description: Which project this belongs to, special chars replaced with dashes
15+
required: true
16+
- name: BRANCH
17+
description: Which branch this belongs to, original value
18+
required: true
19+
- name: PROJECT
20+
description: Which project this belongs to, original value
21+
required: true
22+
- name: LAGOON_GIT_SHA
23+
description: git hash sha of the current deployment
24+
required: true
25+
- name: SERVICE_ROUTER_URL
26+
description: URL of the Router for this service
27+
value: ""
28+
- name: OPENSHIFT_PROJECT
29+
description: Name of the Project that this service is in
30+
required: true
31+
- name: REGISTRY
32+
description: Registry where Images are pushed to
33+
required: true
34+
- name: DEPLOYMENT_STRATEGY
35+
description: Strategy of Deploymentconfig
36+
value: "Rolling"
37+
- name: SERVICE_IMAGE
38+
description: Pullable image of service
39+
required: true
40+
- name: CRONJOBS
41+
description: Oneliner of Cronjobs
42+
value: ""
43+
objects:
44+
- apiVersion: v1
45+
kind: DeploymentConfig
46+
metadata:
47+
creationTimestamp: null
48+
labels:
49+
service: ${SERVICE_NAME}
50+
branch: ${SAFE_BRANCH}
51+
project: ${SAFE_PROJECT}
52+
name: ${SERVICE_NAME}
53+
spec:
54+
replicas: 1
55+
selector:
56+
service: ${SERVICE_NAME}
57+
strategy:
58+
type: ${DEPLOYMENT_STRATEGY}
59+
template:
60+
metadata:
61+
creationTimestamp: null
62+
labels:
63+
service: ${SERVICE_NAME}
64+
branch: ${SAFE_BRANCH}
65+
project: ${SAFE_PROJECT}
66+
spec:
67+
containers:
68+
- image: ${SERVICE_IMAGE}
69+
name: ${SERVICE_NAME}
70+
ports:
71+
- containerPort: 5601
72+
protocol: TCP
73+
readinessProbe:
74+
httpGet:
75+
port: 5601
76+
initialDelaySeconds: 20
77+
livenessProbe:
78+
httpGet:
79+
port: 5601
80+
initialDelaySeconds: 120
81+
envFrom:
82+
- configMapRef:
83+
name: lagoon-env
84+
env:
85+
- name: SERVICE_NAME
86+
value: ${SERVICE_NAME}
87+
- name: CRONJOBS
88+
value: ${CRONJOBS}
89+
- name: LOGSDB_ADMIN_PASSWORD
90+
valueFrom:
91+
secretKeyRef:
92+
key: LOGSDB_ADMIN_PASSWORD
93+
name: logs-db-admin-password
94+
- name: LOGSDB_KIBANASERVER_PASSWORD
95+
valueFrom:
96+
secretKeyRef:
97+
key: LOGSDB_KIBANASERVER_PASSWORD
98+
name: logs-db-kibanaserver-password
99+
resources:
100+
requests:
101+
cpu: 10m
102+
memory: 10Mi
103+
test: false
104+
triggers:
105+
- type: ConfigChange
106+
status: {}

‎services/logs-db-ui/Dockerfile

+14-10
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,28 @@
11
ARG IMAGE_REPO
22
FROM ${IMAGE_REPO:-lagoon}/kibana
33

4-
COPY ./init/ /lagoon/kibana-init
5-
6-
RUN fix-permissions /lagoon/kibana-init
7-
8-
ENV NODE_OPTIONS="--max-old-space-size=2048"
4+
ENV NODE_OPTIONS="--max-old-space-size=2048" \
5+
LOGSDB_KIBANASERVER_PASSWORD=kibanaserver \
6+
LOGSDB_ADMIN_PASSWORD=admin \
7+
ELASTICSEARCH_URL=http://logs-db:9200
98

109
RUN echo $'xpack.security.enabled: false\n\
1110
\n\
1211
# Configure the Kibana internal server user\n\
13-
elasticsearch.username: "admin"\n\
14-
elasticsearch.password: "admin"\n\
12+
elasticsearch.username: "kibanaserver"\n\
13+
elasticsearch.password: "${LOGSDB_KIBANASERVER_PASSWORD}"\n\
1514
\n\
1615
# Disable SSL verification because we use self-signed demo certificates\n\
1716
elasticsearch.ssl.verificationMode: none\n\
1817
\n\
19-
# Whitelist the Search Guard Multi Tenancy Header\n\
18+
# Whitelist the Search Guard Multi Tenancy Header\n\\
2019
elasticsearch.requestHeadersWhitelist: [ "Authorization", "sgtenant" ]' >> config/kibana.yml
2120

22-
RUN bin/kibana-plugin install https://search.maven.org/remotecontent?filepath=com/floragunn/search-guard-kibana-plugin/6.1.1-12/search-guard-kibana-plugin-6.1.1-12.zip
21+
RUN bin/kibana-plugin install https://search.maven.org/remotecontent?filepath=com/floragunn/search-guard-kibana-plugin/6.2.4-13/search-guard-kibana-plugin-6.2.4-13.zip
22+
23+
COPY ./init/ /lagoon/kibana-init
24+
COPY entrypoints/90-kibanayaml-envplate.sh /lagoon/entrypoints/
25+
26+
RUN fix-permissions config/kibana.yml \
27+
&& fix-permissions /lagoon/kibana-init
2328

24-
ENV ELASTICSEARCH_URL=http://logs-db:9200

‎services/logs-db-ui/entrypoints/90-kibanayaml-envplate.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
3+
ep config/kibana.yml

‎services/logs-db-ui/init/index-patterns.sh

+7-7
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,21 @@
11
#!/usr/bin/env bash
22

33
# test for lagoon-logs-* index pattern, create and set to default if it does not exist
4-
until sleep 15; curl --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "lagoon-logs";
4+
until sleep 15; curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "lagoon-logs";
55
do
6-
LAGOON_LOG_ID=$(curl --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"lagoon-logs-*","timeFieldName":"@timestamp"}}' --compressed \
6+
LAGOON_LOG_ID=$(curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"lagoon-logs-*","timeFieldName":"@timestamp"}}' --compressed \
77
| grep -oE '"id":(\d*?,|.*?[^\\]",)' | awk -F'"' '{print $4}') && \
8-
curl 'http://logs-db-ui:5601/api/kibana/settings/defaultIndex' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' -H 'Connection: keep-alive' -H 'DNT: 1' --data-binary "{\"value\":\"$LAGOON_LOG_ID\"}" --compressed
8+
curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/kibana/settings/defaultIndex' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' -H 'Connection: keep-alive' -H 'DNT: 1' --data-binary "{\"value\":\"$LAGOON_LOG_ID\"}" --compressed
99
done
1010

1111
# test for service-logs-* index pattern, create if it does not exist
12-
until curl --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "service-logs";
12+
until curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "service-logs";
1313
do
14-
curl 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"service-logs-*","timeFieldName":"@timestamp"}}' --compressed
14+
curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"service-logs-*","timeFieldName":"@timestamp"}}' --compressed
1515
done
1616

1717
# test for router-logs-* index pattern, create if it does not exist
18-
until curl --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "router-logs";
18+
until curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" --fail --silent 'http://logs-db-ui:5601/api/saved_objects/index-pattern' | grep "router-logs";
1919
do
20-
curl 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"router-logs-*","timeFieldName":"@timestamp"}}' --compressed
20+
curl -u "kibanaserver$LOGSDB_KIBANASERVER_PASSWORD" 'http://logs-db-ui:5601/api/saved_objects/index-pattern' -H 'kbn-version: 6.1.1' -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/json, text/plain, */*' --data-binary '{"attributes":{"title":"router-logs-*","timeFieldName":"@timestamp"}}' --compressed
2121
done

‎services/logs-db-ui/init/watchers.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env bash
22

3-
until sleep 15; curl --silent 'http://logs-db:9200';
3+
until sleep 15; curl -u "admin:$LOGSDB_ADMIN_PASSWORD" --silent 'http://logs-db:9200';
44
do
5-
curl -XPUT "http://logs-db:9200/_xpack/watcher/watch/rabbitmq_connection_error" -H 'Content-Type: application/json' -d @rabbitmq_connection_error.json
5+
curl -u "admin:$LOGSDB_ADMIN_PASSWORD" -XPUT "http://logs-db:9200/_xpack/watcher/watch/rabbitmq_connection_error" -H 'Content-Type: application/json' -d @rabbitmq_connection_error.json
66
done

0 commit comments

Comments
 (0)