Skip to content

Commit

Permalink
Added processing and parsing functionality for skiplist tool
Browse files Browse the repository at this point in the history 
This patch contains everything required for the tool to
process the log files of different releases viz.
[master, train, stein, rocky, queens] and dump the data to
stdout.

Here skiplist.py works like the proccessor and gets all
possible files from get_last_build API of
tempest_file_downloader.py and processes those files
using tempest_html_json.py script.

load_pass_mysql.sql is the sql script for defining
schema required for the tool.

http://localhost:3000/dashboard/snapshot/xdgBIO4fkApkQy4nekCpF4QsoFI7z9C5

Link of snapshot.raintank.io:-
https://snapshot.raintank.io/dashboard/snapshot/qE5NnYG4IIQzC8z9KVOp15SeSel67GWG

Change-Id: I89af8cdd237461c4d0c420d08facbaa461291135
Signed-off-by: Soniya Vyas<[email protected]>
  • Loading branch information
@son-vyas
son-vyas authored and Zuul CI committed Dec 3, 2019
1 parent 79bcc9c commit 347870c
Show file tree
Hide file tree
Showing 8 changed files with 1,245 additions and 15 deletions.
727 changes: 715 additions & 12 deletions ci-scripts/infra-setup/roles/rrcockpit/files/grafana/cockpit.dashboard.json
View file

Large diffs are not rendered by default.

162 changes: 162 additions & 0 deletions ci-scripts/infra-setup/roles/rrcockpit/files/grafana/skiplist_dashboard.dashboard.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
{
"dashboard":{
"annotations":{
"list":[
{
"builtIn":1,
"datasource":"-- Grafana --",
"enable":true,
"hide":true,
"iconColor":"rgba(0, 211, 255, 1)",
"name":"Annotations & Alerts",
"type":"dashboard"
}
]
},
"editable":true,
"gnetId":null,
"graphTooltip":0,
"links":[],
"panels":[
{
"columns":[],
"datasource":"mariadb",
"fontSize":"100%",
"gridPos":{
"h":9,
"w":12,
"x":0,
"y":0
},
"id":2,
"links":[],
"pageSize":null,
"scroll":true,
"showHeader":true,
"sort":{
"col":null,
"desc":false
},
"styles":[
{
"alias":"Time",
"dateFormat":"YYYY-MM-DD HH:mm:ss",
"pattern":"Time",
"type":"date"
},
{
"alias":"",
"colorMode":null,
"colors":[
"rgba(50, 172, 45, 0.97)",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"dateFormat":"YYYY-MM-DD HH:mm:ss",
"decimals":2,
"mappingType":1,
"pattern":"",
"thresholds":[],
"type":"number",
"unit":"short"
},
{
"alias":"",
"colorMode":"row",
"colors":[
"#37872D",
"rgba(237, 129, 40, 0.89)",
"rgba(245, 54, 54, 0.9)"
],
"decimals":0,
"pattern":"/.*/",
"thresholds":[],
"type":"number",
"unit":"short"
}
],
"targets":[
{
"format":"table",
"group":[],
"metricColumn":"Branch",
"rawQuery":true,
"rawSql":"SELECT\n ts AS \"time\",\n Branch ,\n Testname ,\n Result, \n IF(Result = 'Success', count(Testname) , null) as Passed,\n IF(Result = 'Failure', count(Testname), null) as Failed\n FROM rr_pass_tests\ngroup by Testname;\n",
"refId":"A",
"select":[
[
{
"params":[
"Testname"
],
"type":"column"
}
],
[
{
"params":[
"id"
],
"type":"column"
}
]
],
"table":"rr_pass_tests",
"timeColumn":"ts",
"timeColumnType":"timestamp",
"where":[
{
"name":"$__timeFilter",
"params":[],
"type":"macro"
}
]
}
],
"timeFrom":null,
"timeShift":null,
"title":"Skiplist: Needs attention",
"transform":"table",
"transparent":true,
"type":"table"
}
],
"schemaVersion":18,
"style":"dark",
"tags":[],
"templating":{
"list":[]
},
"time":{
"from":"now-6h",
"to":"now"
},
"timepicker":{
"refresh_intervals":[
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options":[
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone":"",
"title":"Skiplist_dashboard"
}
}
17 changes: 17 additions & 0 deletions ci-scripts/infra-setup/roles/rrcockpit/files/mariadb/load_pass_mysql.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
USE rrcockpit;

CREATE TABLE IF NOT EXISTS rr_pass_tests (
id MEDIUMINT NOT NULL AUTO_INCREMENT,
Branch VARCHAR(255) NOT NULL,
Testname VARCHAR(255) NOT NULL,
ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
Result VARCHAR(255) NOT NULL,
PRIMARY KEY (id)
);

TRUNCATE TABLE rr_pass_tests;

LOAD DATA LOCAL INFILE '/tmp/pass.csv' \n
INTO TABLE rr_pass_tests \n
FIELDS TERMINATED BY ',' \n
LINES TERMINATED BY '\n';
5 changes: 3 additions & 2 deletions ci-scripts/infra-setup/roles/rrcockpit/files/mariadb/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#httplib2==0.11.3
#requests==2.19.1
ansible==2.7.6
bs4==0.0.1
beautifulsoup4==4.8.1
diskcache==4.1.0
launchpadlib==1.10.6
python-bugzilla==2.1.0
requests==2.21.0
20 changes: 19 additions & 1 deletion ci-scripts/infra-setup/roles/rrcockpit/files/mariadb/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
set -x

read_lp(){

launchpad_bugs_mariadb.py \
--tag alert \
--status \
Expand Down Expand Up @@ -40,30 +41,45 @@ read_lp(){
launchpad_bugs_mariadb.py \
--status \
New

}

read_recent_lp(){

launchpad_bugs_mariadb.py \
--previous_days=5

}

read_bz(){

bugzilla_bugs_mariadb.py

}

read_pass(){

skiplist.py

}

read_noop(){

releases="master rocky queens pike"
types="upstream rdo tempest"
for release in $releases; do
for type in $types; do
noop_build.py --release $release --type $type
done
done

}

load_mariadb(){

read_$1 > /tmp/$1.csv
mysql -h mariadb -P 3306 -u root < /tmp/load_$1_mysql.sql

}

# We could have just keep sleep 60 before load_db, but this helps dev
Expand All @@ -74,9 +90,11 @@ ansible-playbook /tmp/wait-mariadb.yaml
while true; do
# noop jobs have been disabled
# load_mariadb noop 2>&1 | tee /tmp/run.log
load_mariadb drop 2>&1 | tee /tmp/run.log

load_mariadb pass 2>&1 | tee /tmp/run.log
load_mariadb lp 2>&1 | tee /tmp/run.log
load_mariadb bz 2>&1 | tee /tmp/run.log
load_mariadb recent_lp 2>&1 | tee /tmp/run.log
sleep 1440;

done
74 changes: 74 additions & 0 deletions ci-scripts/infra-setup/roles/rrcockpit/files/mariadb/skiplist.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env python
# Copyright 2019 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import json
import tempest_file_downloader
import tempest_html_json

from datetime import datetime


parse_result = []


def get_files():
"""
This function gives the result of all the files parsed
"""
release_names = ['master', 'train', 'stein', 'rocky', 'queens']
for release_name in release_names:
if release_name in ['master', 'train']:
file_name = "stestr_results.html"
else:
file_name = "tempest.html.gz"
try:
log_url, release = tempest_file_downloader.get_last_build(
"periodic-tripleo-ci-centos-7-ovb-1ctlr_2comp-"
"featureset021-" + release_name, file_name)
global log
log = log_url
except Exception:
continue
parse_result.append(tempest_html_json.output(log, release))
return parse_result


def get_output_file():
"""
This function returns json serialized data
"""
json.dumps(get_files(), sort_keys=True, indent=2)


def print_as_csv():
"""
This function print the result in csv format
"""
timestamp = datetime.now()
for file in get_files():
for result in file.values():
for release_name, test in result.iteritems():
for testname, status in test.iteritems():
print(('{},{},{},{},{}').format(
0,
release_name,
testname.split('(')[-1].replace("']", ""),
str(timestamp),
status))


if __name__ == "__main__":
print_as_csv()
Loading

0 comments on commit 347870c

Please sign in to comment.