Skip to content

Commit fe64d84

Browse files
snisni
committed
rest: add calc and s transformation function
1 parent 5624a20 commit fe64d84

File tree

4 files changed

+137
-16
lines changed

4 files changed

+137
-16
lines changed

‎MANIFEST

+1
Original file line numberDiff line numberDiff line change
@@ -2584,6 +2584,7 @@ t/scenarios/rest_api/t/301-controller_rest_reports.t
25842584
t/scenarios/rest_api/t/301-controller_rest_scenario.t
25852585
t/scenarios/rest_api/t/305-controller_rest_commands.t
25862586
t/scenarios/rest_api/t/local/rest.t
2587+
t/scenarios/rest_api/t/local/rest_api_misc.t
25872588
t/scenarios/rest_api/t/local/rest_check_thruk_rest.t
25882589
t/scenarios/rest_api/t/local/rest_commands.t
25892590
t/scenarios/rest_api/t/local/rest_configtool.t

‎docs/documentation/rest.asciidoc

+3
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,9 @@ Available transformations functions are:
380380
* `lower`: make column lower case
381381
* `lc`: alias for lower
382382
* `substr`: extract substring (substr(columnname, offset [, length]))
383+
* `calc`: apply mathematical operation (calc(column, op , value) ex.: calc(last_check, '*', 1000))
384+
* `s`: apply regex replacement (s(column, regex, replace) ex.: s(host_name, '/\..*$/', '') Note: you cannot use backreferences like $1 in the replacement string for security reasons.
385+
* `unit`: set the unit for this column (unit(column, unit) ex.: unit(calc(rta, "*", 1000), "s")
383386

384387
ex.: return first 3 characters from upper case host name.
385388

‎lib/Thruk/Controller/rest_v1.pm

+95-16
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,11 @@ sub process_rest_request {
184184
}
185185
}
186186

187+
my $wrapped;
188+
$wrapped = 1 if($c->req->header("x-thruk-outputformat") && $c->req->header("x-thruk-outputformat") eq 'wrapped_json');
189+
$wrapped = 1 if(defined $c->req->parameters->{'headers'} && $c->req->parameters->{'headers'} eq 'wrapped_json');
190+
$c->stash->{'meta_column_info'} = {};
191+
187192
if(!$data) {
188193
eval {
189194
$data = _fetch($c, $path_info, $method);
@@ -207,9 +212,6 @@ sub process_rest_request {
207212
}
208213

209214
# add columns meta data, ex.: used in the thruk grafana datasource
210-
my $wrapped;
211-
$wrapped = 1 if($c->req->header("x-thruk-outputformat") && $c->req->header("x-thruk-outputformat") eq 'wrapped_json');
212-
$wrapped = 1 if(defined $c->req->parameters->{'headers'} && $c->req->parameters->{'headers'} eq 'wrapped_json');
213215
if($wrapped) {
214216
# wrap data along with column meta data
215217
my $request_method = $method || $c->req->method;
@@ -785,7 +787,7 @@ sub get_request_columns {
785787
if($_ =~ m/\s+as\s+([^\s]+)\s*$/mx) {
786788
$_ =~ s/^.*?\s+as\s+([^\s]+)\s*$/$1/mx; # strip "as alias" from column
787789
} else {
788-
$_ =~ s/^[^:]+:(.*?)$/$1/gmx; # strip alias
790+
$_ =~ s/^[^:]+:([^"']*?)$/$1/gmx; # strip alias
789791
}
790792
}
791793
}
@@ -794,7 +796,7 @@ sub get_request_columns {
794796
if($_ =~ m/\s+as\s+([^\s]+)\s*$/mx) {
795797
$_ =~ s/\s+as\s+([^\s]+)\s*$//mx; # strip "as alias" from column
796798
} else {
797-
$_ =~ s/^([^:]+):.*?$/$1/gmx; # strip alias
799+
$_ =~ s/^([^:]+):[^"']*?$/$1/gmx; # strip alias
798800
}
799801
$_ =~ s/^.*\(([^)]+)\)$/$1/gmx; # extract column name from aggregation function
800802
}
@@ -821,7 +823,7 @@ sub get_aliased_columns {
821823
if($col =~ m/^([^:]+)\s+as\s+(.*)$/mx) {
822824
$alias_columns->{$2} = $1;
823825
}
824-
elsif($col =~ m/^([^:]+):(.*)$/mx) {
826+
elsif($col =~ m/^([^:]+):([^"']*)$/mx) {
825827
$alias_columns->{$2} = $1;
826828
}
827829
}
@@ -929,15 +931,27 @@ sub _apply_columns {
929931
}
930932
}
931933

934+
# extract helpful meta information from data
935+
my $firstrow;
936+
if($data && $data->[0]) {
937+
$firstrow = $data->[0];
938+
}
939+
for my $col (@{$columns}) {
940+
$c->stash->{'meta_column_info'}->{$col->{'orig'}} = $col;
941+
if($firstrow && $firstrow->{$col->{'column'}."_unit"}) {
942+
$col->{'unit'} = $firstrow->{$col->{'column'}."_unit"};
943+
}
944+
}
945+
932946
my $res = [];
933947
for my $d (@{$data}) {
934948
my $row = {};
935-
for my $c (@{$columns}) {
936-
my $val = $d->{$c->{'orig'}} // $d->{$c->{'column'}};
937-
for my $f (@{$c->{'func'}}) {
938-
$val = _apply_data_function($f, $val);
949+
for my $col (@{$columns}) {
950+
my $val = $d->{$col->{'orig'}} // $d->{$col->{'column'}};
951+
for my $f (@{$col->{'func'}}) {
952+
$val = _apply_data_function($col, $f, $val);
939953
}
940-
$row->{$c->{'alias'}} = $val;
954+
$row->{$col->{'alias'}} = $val;
941955
}
942956
push @{$res}, $row;
943957
}
@@ -947,7 +961,7 @@ sub _apply_columns {
947961

948962
##########################################################
949963
sub _apply_data_function {
950-
my($f, $val) = @_;
964+
my($col, $f, $val) = @_;
951965
my($name, $args) = @{$f};
952966
if($name eq 'lower' || $name eq 'lc') {
953967
return lc($val // '');
@@ -962,7 +976,41 @@ sub _apply_data_function {
962976
if(defined $args->[0]) {
963977
return(substr($val // '', $args->[0]));
964978
}
965-
die("usage: substr(value, start[, length])");
979+
die("usage: substr(column, start[, length])");
980+
}
981+
if($name eq 'calc') {
982+
my $op = _trim_quotes($args->[0]);
983+
my $v = _trim_quotes($args->[1]);
984+
die("usage: calc(column, 'operator', 'value')") if(!defined $op || !defined $v);
985+
if($op eq '*') {
986+
$val *= $v;
987+
} elsif($op eq '/') {
988+
$val = $val / $v;
989+
} elsif($op eq '+' || $op eq ' ' || $op eq '') { # assume space as +, might have been url decoded
990+
$val = $val + $v;
991+
} elsif($op eq '-') {
992+
$val = $val - $v;
993+
} else {
994+
die("unsupported operator, use one of: + - * /");
995+
}
996+
return $val;
997+
}
998+
if($name eq 's') {
999+
my $regexp = _trim_quotes($args->[0]);
1000+
my $replace = _trim_quotes($args->[1]);
1001+
die("usage: s(column, 'regexp', 'replacement')") if(!$regexp || !defined $replace);
1002+
$regexp = _trim_re($regexp);
1003+
## no critic
1004+
my $re = qr($regexp);
1005+
$val =~ s/$re/$replace/g;
1006+
## use critic
1007+
return $val;
1008+
}
1009+
1010+
# just set the unit, do not change the value
1011+
if($name eq 'unit') {
1012+
$col->{'unit'} = _trim_quotes($args->[0]);
1013+
return $val;
9661014
}
9671015

9681016
die("unknown function: ".$name);
@@ -1315,14 +1363,20 @@ sub _get_columns_meta_for_path {
13151363
last;
13161364
}
13171365

1318-
return unless scalar keys %{$columns} > 0;
1319-
13201366
my $meta = [];
13211367
for my $d (@{$req_columns}) {
13221368
my $col = $columns->{$alias_columns->{$d} // $d} // {};
13231369
$col->{'name'} = $d;
1370+
my $hint = $c->stash->{'meta_column_info'}->{$alias_columns->{$d} // $d} // $c->stash->{'meta_column_info'}->{$d};
1371+
if((!defined $col->{'config'} || !defined $col->{'config'}->{'unit'}) && $hint) {
1372+
if(defined $hint->{'unit'}) {
1373+
$col->{'config'}->{'unit'} = $hint->{'unit'};
1374+
$col->{'type'} = 'number' unless $col->{'type'};
1375+
}
1376+
}
13241377
push @{$meta}, $col;
13251378
}
1379+
13261380
return $meta;
13271381
}
13281382

@@ -2498,7 +2552,7 @@ sub _parse_columns_data {
24982552
if($c =~ m/^(.+)\s+as\s+(.*?)$/gmx) {
24992553
$name = $1;
25002554
$alias = $2;
2501-
} elsif($c =~ m/^(.+):(.*?)$/gmx) {
2555+
} elsif($c =~ m/^(.+):([^"']*?)$/gmx) {
25022556
$name = $1;
25032557
$alias = $2;
25042558
}
@@ -2586,6 +2640,31 @@ sub _split_by_comma {
25862640
return @res;
25872641
}
25882642

2643+
##########################################################
2644+
# remove quotes from string
2645+
sub _trim_quotes {
2646+
my($val) = @_;
2647+
return unless defined $val;
2648+
if($val =~ s/^'(.*)'$/$1/mx) {
2649+
return $val;
2650+
}
2651+
if($val =~ s/^"(.*)"$/$1/mx) {
2652+
return $val;
2653+
}
2654+
return $val;
2655+
}
2656+
2657+
##########################################################
2658+
# remove slashes from string
2659+
sub _trim_re {
2660+
my($val) = @_;
2661+
return unless defined $val;
2662+
if($val =~ s|^/(.*)/$|$1|mx) {
2663+
return $val;
2664+
}
2665+
return $val;
2666+
}
2667+
25892668
##########################################################
25902669

25912670
1;

‎t/scenarios/rest_api/t/local/rest_api_misc.t

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
use warnings;
2+
use strict;
3+
use Test::More;
4+
use utf8;
5+
6+
BEGIN {
7+
use lib('t');
8+
require TestUtils;
9+
import TestUtils;
10+
}
11+
plan tests => 20;
12+
13+
###########################################################
14+
# test thruks script path
15+
TestUtils::test_command({
16+
cmd => '/bin/bash -c "type thruk"',
17+
like => ['/\/thruk\/script\/thruk/'],
18+
}) or BAIL_OUT("wrong thruk path");
19+
20+
$ENV{'THRUK_TEST_AUTH_KEY'} = "testkey";
21+
$ENV{'THRUK_TEST_AUTH_USER'} = "omdadmin";
22+
23+
###########################################################
24+
# rest api text transformation
25+
{
26+
TestUtils::test_command({
27+
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=name,calc(rta, "+", 1) as rta_plus&headers=wrapped_json\'',
28+
like => ['/rta_plus/', '/localhost/', '/"ms"/'],
29+
});
30+
TestUtils::test_command({
31+
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=name,unit(calc(rta, "*", 1000), "s") as rta_seconds&headers=wrapped_json\'',
32+
like => ['/rta_seconds/', '/localhost/', '/"s"/'],
33+
});
34+
TestUtils::test_command({
35+
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=substr(name,0,3)\'',
36+
like => ['/"loc"/'],
37+
});
38+
};

0 commit comments

Comments
 (0)