Skip to content

Commit 58bbf97

Browse files
kibanamachinekibanamachine
authored
Merge branch 'main' into younesmln-kibana-config-for-max-number-of-services-per-service-group
2 parents 125eba2 + 4c9d996 commit 58bbf97

File tree

974 files changed

+22967
-12444
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

974 files changed

+22967
-12444
lines changed

‎.buildkite/pipelines/build_api_docs.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
env:
2+
PUBLISH_API_DOCS_CHANGES: 'true'
3+
steps:
4+
- command: .buildkite/scripts/steps/build_api_docs.sh
5+
label: 'Build API Docs'
6+
agents:
7+
queue: n2-4-spot
8+
key: build_api_docs
9+
timeout_in_minutes: 60
10+
retry:
11+
automatic:
12+
- exit_status: '-1'
13+
limit: 3

‎.buildkite/pull_requests.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
"always_require_ci_on_changed": [
3737
"^docs/developer/plugin-list.asciidoc$",
3838
"/plugins/[^/]+/readme\\.(md|asciidoc)$"
39-
]
39+
],
40+
"kibana_versions_check": true
4041
}
4142
]
4243
}

‎.buildkite/scripts/steps/artifacts/cloud.sh‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,8 @@ CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] |
5656
CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")
5757
CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$LOGS")
5858

59-
CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
60-
CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
59+
export CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
60+
export CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
6161

6262
echo "Kibana: $CLOUD_DEPLOYMENT_KIBANA_URL"
6363
echo "ES: $CLOUD_DEPLOYMENT_ELASTICSEARCH_URL"
@@ -68,15 +68,15 @@ function shutdown {
6868
}
6969
trap "shutdown" EXIT
7070

71-
export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').protocol.replace(':', ''))")
72-
export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').hostname)")
73-
export TEST_KIBANA_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_KIBANA_URL').port)")
71+
export TEST_KIBANA_PROTOCOL=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_KIBANA_URL).protocol.replace(':', ''))")
72+
export TEST_KIBANA_HOSTNAME=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_KIBANA_URL).hostname)")
73+
export TEST_KIBANA_PORT=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_KIBANA_URL).port || 443)")
7474
export TEST_KIBANA_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
7575
export TEST_KIBANA_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"
7676

77-
export TEST_ES_PROTOCOL=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').protocol.replace(':', ''))")
78-
export TEST_ES_HOSTNAME=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').hostname)")
79-
export TEST_ES_PORT=$(node -e "console.log(new URL('$CLOUD_DEPLOYMENT_ELASTICSEARCH_URL').port)")
77+
export TEST_ES_PROTOCOL=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_ELASTICSEARCH_URL).protocol.replace(':', ''))")
78+
export TEST_ES_HOSTNAME=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_ELASTICSEARCH_URL).hostname)")
79+
export TEST_ES_PORT=$(node -e "console.log(new URL(process.env.CLOUD_DEPLOYMENT_ELASTICSEARCH_URL).port || 443)")
8080
export TEST_ES_USERNAME="$CLOUD_DEPLOYMENT_USERNAME"
8181
export TEST_ES_PASSWORD="$CLOUD_DEPLOYMENT_PASSWORD"
8282

‎.buildkite/scripts/steps/build_api_docs.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,22 @@ node scripts/build_ts_refs \
1212

1313
echo "--- Build API Docs"
1414
node --max-old-space-size=12000 scripts/build_api_docs
15+
16+
if [[ "${PUBLISH_API_DOCS_CHANGES:-}" == "true" ]]; then
17+
echo "--- Publish API Docs"
18+
19+
git config --global user.name kibanamachine
20+
git config --global user.email '[email protected]'
21+
22+
branch="api_docs_$(date +%F_%H-%M-%S)"
23+
git checkout -b "$branch"
24+
git add ./*.docnav.json
25+
git add api_docs
26+
git commit -m "[api-docs] Daily api_docs build"
27+
28+
git remote add kibanamachine https://github.com/kibanamachine/kibana.git
29+
git push -u kibanamachine "$branch"
30+
prUrl=$(gh pr create --repo elastic/kibana --title "[api-docs] $(date +%F) Daily api_docs build" --body "Generated by $BUILDKITE_BUILD_URL" --label "release_note:skip" --label "backport:auto-version")
31+
echo "Opened PR: $prUrl"
32+
gh pr merge --repo elastic/kibana --auto --squash "$prUrl"
33+
fi

‎.buildkite/scripts/steps/code_coverage/ftr_configs.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ printf "%s\n" "${results[@]}"
116116
echo ""
117117

118118
# So the last step "knows" this config ran
119-
uploadRanFile "ftr_configs"
119+
uploadRanFile "functional"
120120

121121
# Force exit 0 to ensure the next build step starts.
122122
exit 0

‎.buildkite/scripts/steps/code_coverage/ingest.sh‎

Lines changed: 91 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8,59 +8,103 @@ source .buildkite/scripts/steps/code_coverage/merge.sh
88

99
export CODE_COVERAGE=1
1010
echo "--- Reading Kibana stats cluster creds from vault"
11-
export USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)"
12-
export PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)"
13-
export HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
14-
export TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
15-
16-
echo "--- Print KIBANA_DIR"
17-
echo "### KIBANA_DIR: $KIBANA_DIR"
11+
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/prod/coverage/elasticsearch)"
12+
export USER_FROM_VAULT
13+
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/prod/coverage/elasticsearch)"
14+
export PASS_FROM_VAULT
15+
HOST_FROM_VAULT="$(retry 5 5 vault read -field=host secret/kibana-issues/prod/coverage/elasticsearch)"
16+
export HOST_FROM_VAULT
17+
TIME_STAMP=$(date +"%Y-%m-%dT%H:%M:00Z")
18+
export TIME_STAMP
1819

1920
echo "--- Download previous git sha"
2021
.buildkite/scripts/steps/code_coverage/reporting/downloadPrevSha.sh
21-
previousSha=$(cat downloaded_previous.txt)
22+
PREVIOUS_SHA=$(cat downloaded_previous.txt)
2223

2324
echo "--- Upload new git sha"
2425
.buildkite/scripts/steps/code_coverage/reporting/uploadPrevSha.sh
2526

2627
.buildkite/scripts/bootstrap.sh
2728

28-
echo "--- Download coverage artifacts"
29-
buildkite-agent artifact download target/kibana-coverage/jest/* .
30-
#buildkite-agent artifact download target/kibana-coverage/functional/* .
31-
buildkite-agent artifact download target/ran_files/* .
32-
ls -l target/ran_files/* || echo "### No ran-files found"
33-
34-
echo "--- process HTML Links"
35-
.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh
36-
37-
echo "--- collect VCS Info"
38-
.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh
39-
40-
echo "--- Jest: Reset file paths prefix, merge coverage files, and generate the final combined report"
41-
# Jest: Reset file paths prefix to Kibana Dir of final worker
42-
replacePaths "$KIBANA_DIR/target/kibana-coverage/jest" "CC_REPLACEMENT_ANCHOR" "$KIBANA_DIR"
43-
yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js
44-
45-
#echo "--- Functional: Reset file paths prefix, merge coverage files, and generate the final combined report"
46-
# Functional: Reset file paths prefix to Kibana Dir of final worker
47-
#set +e
48-
#sed -ie "s|CC_REPLACEMENT_ANCHOR|${KIBANA_DIR}|g" target/kibana-coverage/functional/*.json
49-
#echo "--- Begin Split and Merge for Functional"
50-
#splitCoverage target/kibana-coverage/functional
51-
#splitMerge
52-
#set -e
53-
54-
echo "--- Archive and upload combined reports"
55-
collectAndUpload target/kibana-coverage/jest/kibana-jest-coverage.tar.gz \
56-
target/kibana-coverage/jest-combined
57-
#collectAndUpload target/kibana-coverage/functional/kibana-functional-coverage.tar.gz \
58-
# target/kibana-coverage/functional-combined
59-
60-
echo "--- Upload coverage static site"
61-
.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh
62-
63-
echo "--- Ingest results to Kibana stats cluster"
64-
.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' \
65-
${BUILDKITE_BUILD_NUMBER} ${BUILDKITE_BUILD_URL} ${previousSha} \
66-
'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
29+
collectRan() {
30+
buildkite-agent artifact download target/ran_files/* .
31+
32+
while read -r x; do
33+
ran=("${ran[@]}" "$(cat "$x")")
34+
done <<<"$(find target/ran_files -maxdepth 1 -type f -name '*.txt')"
35+
36+
echo "--- Collected Ran files: ${ran[*]}"
37+
}
38+
39+
uniqueifyRanConfigs() {
40+
local xs=("$@")
41+
local xss
42+
xss=$(printf "%s\n" "${xs[@]}" | sort -u | tr '\n' ' ' | xargs) # xargs trims whitespace
43+
uniqRanConfigs=("$xss")
44+
echo "--- Uniq Ran files: ${uniqRanConfigs[*]}"
45+
}
46+
47+
fetchArtifacts() {
48+
echo "--- Fetch coverage artifacts"
49+
50+
local xs=("$@")
51+
for x in "${xs[@]}"; do
52+
buildkite-agent artifact download "target/kibana-coverage/${x}/*" .
53+
done
54+
}
55+
56+
archiveReports() {
57+
echo "--- Archive and upload combined reports"
58+
59+
local xs=("$@")
60+
for x in "${xs[@]}"; do
61+
echo "### Collect and Upload for: ${x}"
62+
# fileHeads "target/file-heads-archive-reports-for-${x}.txt" "target/kibana-coverage/${x}"
63+
# dirListing "target/dir-listing-${x}-combined-during-archiveReports.txt" target/kibana-coverage/${x}-combined
64+
# dirListing "target/dir-listing-${x}-during-archiveReports.txt" target/kibana-coverage/${x}
65+
collectAndUpload "target/kibana-coverage/${x}/kibana-${x}-coverage.tar.gz" "target/kibana-coverage/${x}-combined"
66+
done
67+
}
68+
69+
mergeAll() {
70+
local xs=("$@")
71+
72+
for x in "${xs[@]}"; do
73+
if [ "$x" == "jest" ]; then
74+
echo "--- [$x]: Reset file paths prefix, merge coverage files, and generate the final combined report"
75+
replacePaths "$KIBANA_DIR/target/kibana-coverage/jest" "CC_REPLACEMENT_ANCHOR" "$KIBANA_DIR"
76+
yarn nyc report --nycrc-path src/dev/code_coverage/nyc_config/nyc.jest.config.js
77+
elif [ "$x" == "functional" ]; then
78+
echo "---[$x] : Reset file paths prefix, merge coverage files, and generate the final combined report"
79+
set +e
80+
sed -ie "s|CC_REPLACEMENT_ANCHOR|${KIBANA_DIR}|g" target/kibana-coverage/functional/*.json
81+
echo "--- Begin Split and Merge for Functional"
82+
splitCoverage target/kibana-coverage/functional
83+
splitMerge
84+
set -e
85+
fi
86+
done
87+
}
88+
89+
modularize() {
90+
collectRan
91+
if [ -d target/ran_files ]; then
92+
uniqueifyRanConfigs "${ran[@]}"
93+
fetchArtifacts "${uniqRanConfigs[@]}"
94+
mergeAll "${uniqRanConfigs[@]}"
95+
archiveReports "${uniqRanConfigs[@]}"
96+
.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh "${uniqRanConfigs[@]}"
97+
.buildkite/scripts/steps/code_coverage/reporting/uploadStaticSite.sh "${uniqRanConfigs[@]}"
98+
.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh
99+
source .buildkite/scripts/steps/code_coverage/reporting/ingestData.sh 'elastic+kibana+code-coverage' \
100+
"${BUILDKITE_BUILD_NUMBER}" "${BUILDKITE_BUILD_URL}" "${PREVIOUS_SHA}" \
101+
'src/dev/code_coverage/ingest_coverage/team_assignment/team_assignments.txt'
102+
ingestModular "${uniqRanConfigs[@]}"
103+
else
104+
echo "--- Found zero configs that ran, cancelling ingestion."
105+
exit 11
106+
fi
107+
}
108+
109+
modularize
110+
echo "### unique ran configs: ${uniqRanConfigs[*]}"

‎.buildkite/scripts/steps/code_coverage/jest_integration.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ echo '--- Jest Integration code coverage'
1515
.buildkite/scripts/steps/code_coverage/jest_parallel.sh jest.integration.config.js
1616

1717
# So the last step "knows" this config ran
18-
uploadRanFile "jest_integration"
18+
uploadRanFile "jest"

‎.buildkite/scripts/steps/code_coverage/reporting/collectVcsInfo.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
set -euo pipefail
44

5+
echo "--- collect VCS Info"
6+
57
echo "### Prok'd Index File: ..."
68
cat src/dev/code_coverage/www/index.html
79

@@ -27,4 +29,4 @@ for X in "${!XS[@]}"; do
2729
}
2830
done
2931
echo "### VCS_INFO:"
30-
cat VCS_INFO.txt
32+
cat VCS_INFO.txt

‎.buildkite/scripts/steps/code_coverage/reporting/ingestData.sh‎

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
set -euo pipefail
44

5-
echo "### Ingesting Code Coverage"
6-
echo ""
7-
85
COVERAGE_JOB_NAME=$1
96
export COVERAGE_JOB_NAME
107
echo "### debug COVERAGE_JOB_NAME: ${COVERAGE_JOB_NAME}"
@@ -31,27 +28,25 @@ echo "### debug TEAM_ASSIGN_PATH: ${TEAM_ASSIGN_PATH}"
3128

3229
BUFFER_SIZE=500
3330
export BUFFER_SIZE
34-
echo "### debug BUFFER_SIZE: ${BUFFER_SIZE}"
35-
36-
# Build team assignments file
37-
echo "### Generate Team Assignments"
38-
CI_STATS_DISABLED=true node scripts/generate_team_assignments.js \
39-
--verbose --src '.github/CODEOWNERS' --dest $TEAM_ASSIGN_PATH
40-
41-
#for x in functional jest; do
42-
# echo "### Ingesting coverage for ${x}"
43-
# COVERAGE_SUMMARY_FILE="target/kibana-coverage/${x}-combined/coverage-summary.json"
44-
#
45-
# CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} \
46-
# --vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH &
47-
#done
48-
#wait
49-
50-
echo "### Ingesting coverage for JEST"
51-
COVERAGE_SUMMARY_FILE="target/kibana-coverage/jest-combined/coverage-summary.json"
52-
53-
CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path ${COVERAGE_SUMMARY_FILE} \
54-
--vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath $TEAM_ASSIGN_PATH
55-
56-
echo "--- Ingesting Code Coverage - Complete"
57-
echo ""
31+
32+
ingestModular() {
33+
local xs=("$@")
34+
35+
echo "--- Generate Team Assignments"
36+
CI_STATS_DISABLED=true node scripts/generate_team_assignments.js \
37+
--verbose --src '.github/CODEOWNERS' --dest "$TEAM_ASSIGN_PATH"
38+
39+
echo "--- Ingest results to Kibana stats cluster"
40+
for x in "${xs[@]}"; do
41+
echo "--- Ingesting coverage for ${x}"
42+
43+
COVERAGE_SUMMARY_FILE="target/kibana-coverage/${x}-combined/coverage-summary.json"
44+
45+
CI_STATS_DISABLED=true node scripts/ingest_coverage.js --path "${COVERAGE_SUMMARY_FILE}" \
46+
--vcsInfoPath ./VCS_INFO.txt --teamAssignmentsPath "$TEAM_ASSIGN_PATH" &
47+
done
48+
wait
49+
50+
echo "--- Ingesting Code Coverage - Complete"
51+
echo ""
52+
}

‎.buildkite/scripts/steps/code_coverage/reporting/prokLinks.sh‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,20 @@
22

33
set -euo pipefail
44

5-
cat << EOF > src/dev/code_coverage/www/index_partial_2.html
6-
<a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/jest-combined/index.html">Latest Jest</a>
5+
echo "--- process HTML Links"
6+
7+
xs=("$@")
8+
len=${#xs[@]}
9+
10+
# TODO-TRE: Maybe use more exhaustive logic instead of just length.
11+
if [[ $len -eq 2 ]]; then
12+
links="<a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/jest-combined/index.html">Latest Jest</a><a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/functional-combined/index.html">Latest FTR</a>"
13+
else
14+
links="<a class="nav-link" href="https://kibana-coverage.elastic.dev/${TIME_STAMP}/jest-combined/index.html">Latest Jest</a>"
15+
fi
16+
17+
cat <<EOF >src/dev/code_coverage/www/index_partial_2.html
18+
${links}
719
</nav>
820
</div>
921
</header>

0 commit comments

Comments
 (0)