diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go
index e4272936e9..1d5823b726 100644
--- a/src/core/metrics/metrics_util.go
+++ b/src/core/metrics/metrics_util.go
@@ -203,8 +203,17 @@ func GetCalculationMap() map[string]string {
"nginx.http.conn.current": "avg",
"nginx.http.conn.dropped": "sum",
"nginx.http.conn.idle": "avg",
+ "nginx.cache.bypass": "sum",
+ "nginx.cache.expired": "sum",
+ "nginx.cache.hit": "sum",
+ "nginx.cache.miss": "sum",
+ "nginx.cache.revalidated": "sum",
+ "nginx.cache.stale": "sum",
+ "nginx.cache.updating": "sum",
"nginx.upstream.response.buffered": "sum",
"nginx.upstream.request.failed": "sum",
+ "nginx.upstream.request.count": "sum",
+ "nginx.upstream.next.count": "sum",
"nginx.upstream.response.failed": "sum",
"nginx.workers.count": "avg",
"nginx.workers.rlimit_nofile": "avg",
diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go
index 3f03661500..4bcd71b790 100644
--- a/src/core/metrics/sources/nginx_access_log.go
+++ b/src/core/metrics/sources/nginx_access_log.go
@@ -30,6 +30,32 @@ const (
pattern = `[A-Z]+\s.+\s[A-Z]+/.+`
)
+var logVarMap = map[string]string{
+ "$remote_addr": "%{IPORHOST:remote_addr}",
+ "$remote_user": "%{USERNAME:remote_user}",
+ "$time_local": `%{HTTPDATE:time_local}`,
+ "$status": "%{INT:status}",
+ "$body_bytes_sent": "%{NUMBER:body_bytes_sent}",
+ "$http_referer": "%{DATA:http_referer}",
+ "$http_user_agent": "%{DATA:http_user_agent}",
+ "$http_x_forwarded_for": "%{DATA:http_x_forwarded_for}",
+ "$bytes_sent": "%{NUMBER:bytes_sent}",
+ "$gzip_ratio": "%{DATA:gzip_ratio}",
+ "$server_protocol": "%{DATA:server_protocol}",
+ "$request_length": "%{INT:request_length}",
+ "$request_time": "%{DATA:request_time}",
+ "\"$request\"": "\"%{DATA:request}\"",
+ "$request ": "%{DATA:request} ",
+ "$upstream_connect_time": "%{DATA:upstream_connect_time}",
+ "$upstream_header_time": "%{DATA:upstream_header_time}",
+ "$upstream_response_time": "%{DATA:upstream_response_time}",
+ "$upstream_response_length": "%{DATA:upstream_response_length}",
+ "$upstream_status": "%{DATA:upstream_status}",
+ "$upstream_cache_status": "%{DATA:upstream_cache_status}",
+ "[": "\\[",
+ "]": "\\]",
+}
+
// This metrics source is used to tail the NGINX access logs to retrieve metrics.
type NginxAccessLog struct {
@@ -156,11 +182,16 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St
}
func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) {
- logPattern := convertLogFormat(logFormat)
+ logPattern := logFormat
+
+ for key, value := range logVarMap {
+ logPattern = strings.ReplaceAll(logPattern, key, value)
+ }
+
log.Debugf("Collecting from: %s using format: %s", logFile, logFormat)
log.Debugf("Pattern used for tailing logs: %s", logPattern)
- httpCounters, upstreamCounters := getDefaultCounters()
+ httpCounters, upstreamCounters, upstreamCacheCounters := getDefaultCounters()
gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes := []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}
mu := sync.Mutex{}
@@ -180,53 +211,31 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
select {
case d := <-data:
access, err := tailer.NewNginxAccessItem(d)
+ upstreamRequest := false
if err != nil {
c.logger.Log(fmt.Sprintf("Error decoding access log entry, %v", err))
continue
}
mu.Lock()
- if access.BodyBytesSent != "" {
- if v, err := strconv.Atoi(access.BodyBytesSent); err == nil {
- n := "request.body_bytes_sent"
- httpCounters[n] = float64(v) + httpCounters[n]
- } else {
- c.logger.Log(fmt.Sprintf("Error getting body_bytes_sent value from access logs, %v", err))
- }
- }
- if access.BytesSent != "" {
- if v, err := strconv.Atoi(access.BytesSent); err == nil {
- n := "request.bytes_sent"
- httpCounters[n] = float64(v) + httpCounters[n]
- } else {
- c.logger.Log(fmt.Sprintf("Error getting bytes_sent value from access logs, %v", err))
- }
- }
+ httpCounters = c.parseAccessLogFloatCounters("request.body_bytes_sent", access.BodyBytesSent, httpCounters)
- if access.GzipRatio != "-" && access.GzipRatio != "" {
- if v, err := strconv.Atoi(access.GzipRatio); err == nil {
- gzipRatios = append(gzipRatios, float64(v))
- } else {
- c.logger.Log(fmt.Sprintf("Error getting gzip_ratio value from access logs, %v", err))
- }
- }
+ httpCounters = c.parseAccessLogFloatCounters("request.bytes_sent", access.BytesSent, httpCounters)
- if access.RequestLength != "" {
- if v, err := strconv.Atoi(access.RequestLength); err == nil {
- requestLengths = append(requestLengths, float64(v))
- } else {
- c.logger.Log(fmt.Sprintf("Error getting request_length value from access logs, %v", err))
- }
- }
+ gzipRatios = c.parseAccessLogFloatTimes("gzip_ratio", access.GzipRatio, gzipRatios)
- if access.RequestTime != "" {
- if v, err := strconv.ParseFloat(access.RequestTime, 64); err == nil {
- requestTimes = append(requestTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting request_time value from access logs, %v", err))
- }
- }
+ requestLengths = c.parseAccessLogFloatTimes("request_length", access.RequestLength, requestLengths)
+
+ requestTimes = c.parseAccessLogFloatTimes("request_time", access.RequestTime, requestTimes)
+
+ upstreamConnectTimes = c.parseAccessLogUpstream("upstream_connect_time", access.UpstreamConnectTime, upstreamConnectTimes)
+
+ upstreamHeaderTimes = c.parseAccessLogUpstream("upstream_header_time", access.UpstreamHeaderTime, upstreamHeaderTimes)
+
+ upstreamResponseLength = c.parseAccessLogUpstream("upstream_response_length", access.UpstreamResponseLength, upstreamResponseLength)
+
+ upstreamResponseTimes = c.parseAccessLogUpstream("upstream_response_time", access.UpstreamResponseTime, upstreamResponseTimes)
if access.Request != "" {
method, _, protocol := getParsedRequest(access.Request)
@@ -237,86 +246,47 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
httpCounters[n] = httpCounters[n] + 1
if access.ServerProtocol == "" {
- if strings.Count(protocol, "/") == 1 {
- httpProtocolVersion := strings.Split(protocol, "/")[1]
- httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
- n = fmt.Sprintf("v%s", httpProtocolVersion)
- httpCounters[n] = httpCounters[n] + 1
- }
- }
- }
-
- if access.UpstreamConnectTime != "-" && access.UpstreamConnectTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamConnectTime, 64); err == nil {
- upstreamConnectTimes = append(upstreamConnectTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_connect_time value from access logs, %v", err))
+ calculateServerProtocol(protocol, httpCounters)
}
}
- if access.UpstreamHeaderTime != "-" && access.UpstreamHeaderTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamHeaderTime, 64); err == nil {
- upstreamHeaderTimes = append(upstreamHeaderTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_header_time value from access logs, %v", err))
- }
+ if access.ServerProtocol != "" {
+ calculateServerProtocol(access.ServerProtocol, httpCounters)
}
- if access.UpstreamResponseLength != "-" && access.UpstreamResponseLength != "" {
- if v, err := strconv.ParseFloat(access.UpstreamResponseLength, 64); err == nil {
- upstreamResponseLength = append(upstreamResponseLength, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_response_length value from access logs, %v", err))
+ if access.UpstreamStatus != "" && access.UpstreamStatus != "-" {
+ upstreamRequest = true
+ statusValues := strings.Split(access.UpstreamStatus, ",")
+ for _, value := range statusValues {
+ if v, err := strconv.Atoi(value); err == nil {
+ n := fmt.Sprintf("upstream.status.%dxx", v/100)
+ upstreamCounters[n] = upstreamCounters[n] + 1
+ } else {
+ log.Debugf("Error getting upstream status value from access logs, %v", err)
+ }
}
}
- if access.UpstreamResponseTime != "-" && access.UpstreamResponseTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamResponseTime, 64); err == nil {
- upstreamResponseTimes = append(upstreamResponseTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_response_time value from access logs, %v", err))
- }
+ if access.UpstreamCacheStatus != "" && access.UpstreamCacheStatus != "-" {
+ upstreamRequest = true
+ calculateUpstreamCacheStatus(access.UpstreamCacheStatus, upstreamCacheCounters)
}
- if access.ServerProtocol != "" {
- if strings.Count(access.ServerProtocol, "/") == 1 {
- httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1]
- httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
- n := fmt.Sprintf("v%s", httpProtocolVersion)
- httpCounters[n] = httpCounters[n] + 1
- }
+ // don't need the http status for NGINX Plus
+ if c.nginxType == OSSNginxType {
+ c.calculateHttpStatus(access.Status, httpCounters)
}
- if access.UpstreamStatus != "" && access.UpstreamStatus != "-" {
- if v, err := strconv.Atoi(access.UpstreamStatus); err == nil {
- n := fmt.Sprintf("upstream.status.%dxx", v/100)
- upstreamCounters[n] = upstreamCounters[n] + 1
- } else {
- log.Debugf("Error getting upstream status value from access logs, %v", err)
- }
+ if access.UpstreamConnectTime != "" || access.UpstreamHeaderTime != "" || access.UpstreamResponseTime != "" {
+ upstreamTimes := []string{access.UpstreamConnectTime, access.UpstreamHeaderTime, access.UpstreamResponseTime}
+ upstreamRequest, upstreamCounters = calculateUpstreamNextCount(upstreamTimes, upstreamCounters)
}
- // don't need the http status for NGINX Plus
- if c.nginxType == OSSNginxType {
- if v, err := strconv.Atoi(access.Status); err == nil {
- n := fmt.Sprintf("status.%dxx", v/100)
- httpCounters[n] = httpCounters[n] + 1
- if v == 403 || v == 404 || v == 500 || v == 502 || v == 503 || v == 504 {
- n := fmt.Sprintf("status.%d", v)
- httpCounters[n] = httpCounters[n] + 1
- }
- if v == 499 {
- n := "status.discarded"
- httpCounters[n] = httpCounters[n] + 1
- }
- if v == 400 {
- n := "request.malformed"
- httpCounters[n] = httpCounters[n] + 1
- }
- } else {
- c.logger.Log(fmt.Sprintf("Error getting status value from access logs, %v", err))
- }
+
+ if upstreamRequest == true {
+ upstreamCounters["upstream.request.count"] = upstreamCounters["upstream.request.count"] + 1
}
+
mu.Unlock()
case <-tick.C:
@@ -334,19 +304,19 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
}
if len(requestTimes) > 0 {
- getTimeMetricsMap("request.time", requestTimes, httpCounters)
+ calculateTimeMetricsMap("request.time", requestTimes, httpCounters)
}
if len(upstreamConnectTimes) > 0 {
- getTimeMetricsMap("upstream.connect.time", upstreamConnectTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.connect.time", upstreamConnectTimes, upstreamCounters)
}
if len(upstreamHeaderTimes) > 0 {
- getTimeMetricsMap("upstream.header.time", upstreamHeaderTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.header.time", upstreamHeaderTimes, upstreamCounters)
}
if len(upstreamResponseTimes) > 0 {
- getTimeMetricsMap("upstream.response.time", upstreamResponseTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.response.time", upstreamResponseTimes, upstreamCounters)
}
if len(upstreamResponseLength) > 0 {
@@ -358,10 +328,13 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
c.group = ""
simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(upstreamCounters)...)
+ c.group = ""
+ simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(upstreamCacheCounters)...)
+
log.Tracef("Access log metrics collected: %v", simpleMetrics)
// reset the counters
- httpCounters, upstreamCounters = getDefaultCounters()
+ httpCounters, upstreamCounters, upstreamCacheCounters = getDefaultCounters()
gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}
c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics))
@@ -379,6 +352,72 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
}
}
+func calculateUpstreamNextCount(metricValues []string, upstreamCounters map[string]float64) (bool, map[string]float64) {
+ upstreamRequest := false
+ for _, upstreamTimes := range metricValues {
+ if upstreamTimes != "" && upstreamTimes != "-" {
+ upstreamRequest = true
+ times := strings.Split(upstreamTimes, ", ")
+ if len(times) > 1 {
+ upstreamCounters["upstream.next.count"] = upstreamCounters["upstream.next.count"] + (float64(len(times)) - 1)
+ return upstreamRequest, upstreamCounters
+ }
+ }
+ }
+ return upstreamRequest, upstreamCounters
+}
+
+func (c *NginxAccessLog) parseAccessLogFloatTimes(metricName string, metric string, counter []float64) []float64 {
+ if metric != "" && metric != "-" {
+ if v, err := strconv.ParseFloat(metric, 64); err == nil {
+ counter = append(counter, v)
+ return counter
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ return counter
+}
+
+func (c *NginxAccessLog) parseAccessLogUpstream(metricName string, metric string, counter []float64) []float64 {
+ if metric != "" && metric != "-" {
+ metricValues := strings.Split(metric, ", ")
+ for _, value := range metricValues {
+ if value != "" && value != "-" {
+ if v, err := strconv.ParseFloat(value, 64); err == nil {
+ counter = append(counter, v)
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ }
+ return counter
+
+ }
+ return counter
+}
+
+func (c *NginxAccessLog) parseAccessLogFloatCounters(metricName string, metric string, counters map[string]float64) map[string]float64 {
+ if metric != "" {
+ if v, err := strconv.ParseFloat(metric, 64); err == nil {
+ counters[metricName] = v + counters[metricName]
+ return counters
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ return counters
+}
+
+func calculateServerProtocol(protocol string, counters map[string]float64) {
+ if strings.Count(protocol, "/") == 1 {
+ httpProtocolVersion := strings.Split(protocol, "/")[1]
+ httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
+ n := fmt.Sprintf("v%s", httpProtocolVersion)
+ counters[n] = counters[n] + 1
+ }
+}
+
func getParsedRequest(request string) (method string, uri string, protocol string) {
// Looking for capital letters, a space, anything, a space, capital letters, forward slash then anything.
@@ -436,7 +475,39 @@ func getAverageMetricValue(metricValues []float64) float64 {
return value
}
-func getTimeMetricsMap(metricName string, times []float64, counter map[string]float64) {
+func (c *NginxAccessLog) calculateHttpStatus(status string, counter map[string]float64) {
+
+ if v, err := strconv.Atoi(status); err == nil {
+ n := fmt.Sprintf("status.%dxx", v/100)
+ counter[n] = counter[n] + 1
+ switch v {
+ case 403, 404, 500, 502, 503, 504:
+ n := fmt.Sprintf("status.%d", v)
+ counter[n] = counter[n] + 1
+ case 499:
+ n := "status.discarded"
+ counter[n] = counter[n] + 1
+ case 400:
+ n := "request.malformed"
+ counter[n] = counter[n] + 1
+ }
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting status value from access logs, %v", err))
+ }
+}
+
+func calculateUpstreamCacheStatus(status string, counter map[string]float64) {
+
+ n := fmt.Sprintf("cache.%s", strings.ToLower(status))
+
+ switch status {
+ case "BYPASS", "EXPIRED", "HIT", "MISS", "REVALIDATED", "STALE", "UPDATING":
+ counter[n] = counter[n] + 1
+ return
+ }
+}
+
+func calculateTimeMetricsMap(metricName string, times []float64, counter map[string]float64) {
timeMetrics := map[string]float64{
metricName: 0,
@@ -456,33 +527,6 @@ func getTimeMetricsMap(metricName string, times []float64, counter map[string]fl
}
-// convertLogFormat converts log format into a pattern that can be parsed by the tailer
-func convertLogFormat(logFormat string) string {
- newLogFormat := strings.ReplaceAll(logFormat, "$remote_addr", "%{IPORHOST:remote_addr}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$remote_user", "%{USERNAME:remote_user}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$time_local", `%{HTTPDATE:time_local}`)
- newLogFormat = strings.ReplaceAll(newLogFormat, "$status", "%{INT:status}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$body_bytes_sent", "%{NUMBER:body_bytes_sent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_referer", "%{DATA:http_referer}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_user_agent", "%{DATA:http_user_agent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_x_forwarded_for", "%{DATA:http_x_forwarded_for}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$bytes_sent", "%{NUMBER:bytes_sent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$gzip_ratio", "%{DATA:gzip_ratio}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$server_protocol", "%{DATA:server_protocol}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request_length", "%{INT:request_length}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request_time", "%{DATA:request_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_header_time", "%{DATA:upstream_header_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_response_time", "%{DATA:upstream_response_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_response_length", "%{DATA:upstream_response_length}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_status", "%{DATA:upstream_status}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[")
- newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]")
- return newLogFormat
-}
-
func isOtherMethod(method string) bool {
return method != "method.post" &&
method != "method.get" &&
@@ -492,7 +536,7 @@ func isOtherMethod(method string) bool {
method != "method.options"
}
-func getDefaultCounters() (map[string]float64, map[string]float64) {
+func getDefaultCounters() (map[string]float64, map[string]float64, map[string]float64) {
httpCounters := map[string]float64{
"gzip.ratio": 0,
"method.delete": 0,
@@ -540,6 +584,8 @@ func getDefaultCounters() (map[string]float64, map[string]float64) {
"upstream.header.time.max": 0,
"upstream.header.time.median": 0,
"upstream.header.time.pctl95": 0,
+ "upstream.request.count": 0,
+ "upstream.next.count": 0,
"upstream.response.time": 0,
"upstream.response.time.count": 0,
"upstream.response.time.max": 0,
@@ -553,5 +599,15 @@ func getDefaultCounters() (map[string]float64, map[string]float64) {
"upstream.status.5xx": 0,
}
- return httpCounters, upstreamCounters
+ upstreamCacheCounters := map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 0,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ }
+
+ return httpCounters, upstreamCounters, upstreamCacheCounters
}
diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go
index d272a4a38c..e4b35a00ee 100644
--- a/src/core/metrics/sources/nginx_access_log_test.go
+++ b/src/core/metrics/sources/nginx_access_log_test.go
@@ -68,6 +68,691 @@ func TestAccessLogStop(t *testing.T) {
assert.Len(t, nginxAccessLog.logs, 0)
}
+func TestCalculateUpstreamNextCount(t *testing.T) {
+ upstreamRequest := false
+
+ tests := []struct {
+ name string
+ upstreamTimes []string
+ expectedUpstreamRequest bool
+ expectedCount float64
+ upstreamCounters map[string]float64
+ }{
+ {
+ "singleUpstreamTimes",
+ []string{"0.01", "0.02", "0.00"},
+ true,
+ float64(0),
+ map[string]float64{
+ "upstream.next.count": 0,
+ },
+ },
+ {
+ "multipleUpstreamTimes",
+ []string{"0.01, 0.04, 0.03, 0.00", "0.02, 0.01, 0.03, 0.04", "0.00, 0.00, 0.08, 0.02"},
+ true,
+ float64(3),
+ map[string]float64{
+ "upstream.next.count": 0,
+ },
+ },
+ {
+ "noUpstreamTimes",
+ []string{"-", "-", "-"},
+ false,
+ float64(0),
+ map[string]float64{
+ "upstream.next.count": 0,
+ },
+ },
+ {
+ "emptyUpstreamTimes",
+ []string{"", "", ""},
+ false,
+ float64(0),
+ map[string]float64{
+ "upstream.next.count": 0,
+ },
+ },
+ {
+ "oneUpstreamTime",
+ []string{"-, -, -, 0.04", "-, -, -, 0.02", "-, -, -, 0.02"},
+ true,
+ float64(3),
+ map[string]float64{
+ "upstream.next.count": 0,
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ upstreamRequest, test.upstreamCounters = calculateUpstreamNextCount(test.upstreamTimes, test.upstreamCounters)
+ assert.Equal(t, test.expectedUpstreamRequest, upstreamRequest)
+ assert.Equal(t, test.expectedCount, test.upstreamCounters["upstream.next.count"])
+ })
+
+ }
+
+}
+
+func TestParseAccessLogFloatTimes(t *testing.T) {
+ tests := []struct {
+ name string
+ metricName string
+ metric string
+ counter []float64
+ expectedCounter []float64
+ }{
+ {
+ "validTime",
+ "request_time",
+ "0.00",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03, 0.00},
+ },
+ {
+ "noTime",
+ "request_time",
+ "-",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ {
+ "emptyTime",
+ "request_time",
+ "",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ {
+ "invalidTime",
+ "request_time",
+ "test",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ }
+
+ binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{})
+ collectionDuration := time.Millisecond * 300
+ nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ test.counter = nginxAccessLog.parseAccessLogFloatTimes(test.metricName, test.metric, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+
+ })
+ }
+}
+
+func TestParseAccessLogUpstream(t *testing.T) {
+ tests := []struct {
+ name string
+ metricName string
+ metric string
+ counter []float64
+ expectedCounter []float64
+ }{
+ {
+ "singleTime",
+ "upstream_connect_time",
+ "0.03",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03, 0.03},
+ },
+ {
+ "multipleTimes",
+ "upstream_connect_time",
+ "0.03, 0.04, 0.06",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03, 0.03, 0.04, 0.06},
+ },
+ {
+ "someEmptyTimes",
+ "upstream_connect_time",
+ "0.03, 0.04, -, 0.06, -",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03, 0.03, 0.04, 0.06},
+ },
+ {
+ "emptyTime",
+ "upstream_connect_time",
+ "",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ {
+ "noTime",
+ "upstream_connect_time",
+ "-",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ {
+ "invalidTimes",
+ "upstream_connect_time",
+ "test",
+ []float64{0.02, 0.03},
+ []float64{0.02, 0.03},
+ },
+ }
+
+ binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{})
+ collectionDuration := time.Millisecond * 300
+ nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ test.counter = nginxAccessLog.parseAccessLogUpstream(test.metricName, test.metric, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+
+ })
+ }
+}
+
+func TestParseAccessLogFloatCounters(t *testing.T) {
+ tests := []struct {
+ name string
+ metricName string
+ metric string
+ counter map[string]float64
+ expectedCounter map[string]float64
+ }{
+ {
+
+ "validCount",
+ "request.bytes_sent",
+ "28",
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ map[string]float64{
+ "request.bytes_sent": 32,
+ },
+ },
+ {
+
+ "noCount",
+ "request.bytes_sent",
+ "-",
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ },
+ {
+
+ "emptyCount",
+ "request.bytes_sent",
+ "",
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ },
+ {
+
+ "invalidCount",
+ "request.bytes_sent",
+ "test",
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ map[string]float64{
+ "request.bytes_sent": 4,
+ },
+ },
+ }
+
+ binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{})
+ collectionDuration := time.Millisecond * 300
+ nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ nginxAccessLog.parseAccessLogFloatCounters(test.metricName, test.metric, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+
+ })
+ }
+}
+
+func TestCalculateServerProtocol(t *testing.T) {
+ tests := []struct {
+ name string
+ protocol string
+ counters map[string]float64
+ expectedCounter map[string]float64
+ }{
+ {
+ "validProtocol",
+ "HTTP/1.1",
+ map[string]float64{
+ "v0_9": 0,
+ "v1_0": 0,
+ "v1_1": 2,
+ "v2": 0,
+ },
+ map[string]float64{
+ "v0_9": 0,
+ "v1_0": 0,
+ "v1_1": 3,
+ "v2": 0,
+ },
+ },
+ {
+ "invalidProtocol",
+ "",
+ map[string]float64{
+ "v0_9": 0,
+ "v1_0": 0,
+ "v1_1": 0,
+ "v2": 0,
+ },
+ map[string]float64{
+ "v0_9": 0,
+ "v1_0": 0,
+ "v1_1": 0,
+ "v2": 0,
+ },
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ calculateServerProtocol(test.protocol, test.counters)
+ assert.Equal(t, test.expectedCounter, test.counters)
+ })
+
+ }
+}
+
+func TestGetParsedRequest(t *testing.T) {
+ tests := []struct {
+ name string
+ request string
+ expectedMethod string
+ expectedURI string
+ expectedProtocol string
+ }{
+ {
+ "validRequest",
+ "GET /user/register?ahrefp' or ' HTTP/1.1",
+ "GET",
+ "/user/register?ahrefp' or '",
+ "HTTP/1.1",
+ },
+ {
+ "emptyRequest",
+ "",
+ "",
+ "",
+ "",
+ },
+ {
+ "invalidRequest",
+ "GET /user/register?ahrefp' or ' HTTP1.1",
+ "",
+ "",
+ "",
+ },
+ {
+ "nospacesRequest",
+ "GET/user/register?ahrefp'or'HTTP1.1",
+ "",
+ "",
+ "",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ method, uri, protocol := getParsedRequest(test.request)
+ assert.Equal(t, test.expectedMethod, method)
+ assert.Equal(t, test.expectedURI, uri)
+ assert.Equal(t, test.expectedProtocol, protocol)
+ })
+
+ }
+
+}
+
+func TestGetAverageMetricValue(t *testing.T) {
+ tests := []struct {
+ name string
+ metricValues []float64
+ expectedAverage float64
+ }{
+ {
+ "validValues",
+ []float64{28, 28, 4, 28, 19, 0},
+ 17.833333333333332,
+ },
+ {
+ "emptyValues",
+ []float64{},
+ 0.0,
+ },
+ {
+ "zeroValues",
+ []float64{0, 0, 0, 0},
+ 0.0,
+ },
+ {
+ "decimalValues",
+ []float64{0.02, 0.3, 0.06, 0.07},
+ 0.1125,
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ average := getAverageMetricValue(test.metricValues)
+ assert.Equal(t, test.expectedAverage, average)
+ })
+
+ }
+}
+
+func TestCalculateHTTPStatus(t *testing.T) {
+ tests := []struct {
+ name string
+ status string
+ counter map[string]float64
+ expectedCounter map[string]float64
+ }{
+ {
+ "validStatus",
+ "403",
+ map[string]float64{
+ "status.403": 4,
+ "status.404": 2,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 6,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ map[string]float64{
+ "status.403": 5,
+ "status.404": 2,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 7,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ },
+ {
+ "discardedStatus",
+ "499",
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 0,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 1,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 1,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ },
+ {
+ "malformedStatus",
+ "400",
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 0,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 1,
+ "status.5xx": 0,
+ "request.malformed": 1,
+ },
+ },
+ {
+ "emptyStatus",
+ "",
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 0,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ map[string]float64{
+ "status.403": 0,
+ "status.404": 0,
+ "status.500": 0,
+ "status.502": 0,
+ "status.503": 0,
+ "status.504": 0,
+ "status.discarded": 0,
+ "status.1xx": 0,
+ "status.2xx": 0,
+ "status.3xx": 0,
+ "status.4xx": 0,
+ "status.5xx": 0,
+ "request.malformed": 0,
+ },
+ },
+ }
+
+ binary := core.NewNginxBinary(tutils.NewMockEnvironment(), &config.Config{})
+ collectionDuration := time.Millisecond * 300
+ nginxAccessLog := NewNginxAccessLog(&metrics.CommonDim{}, OSSNamespace, binary, OSSNginxType, collectionDuration)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ nginxAccessLog.calculateHttpStatus(test.status, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+
+ })
+ }
+}
+
+func TestCalculateUpstreamCacheStatus(t *testing.T) {
+ tests := []struct {
+ name string
+ status string
+ counter map[string]float64
+ expectedCounter map[string]float64
+ }{
+ {
+ "validStatus",
+ "HIT",
+ map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 4,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ },
+ map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 5,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ },
+ },
+ {
+ "invalidStatus",
+ "",
+ map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 4,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ },
+ map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 4,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ calculateUpstreamCacheStatus(test.status, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+ })
+
+ }
+}
+
+func TestGetTimeMetricsMap(t *testing.T) {
+ tests := []struct {
+ name string
+ metricName string
+ metricTimes []float64
+ counter map[string]float64
+ expectedCounter map[string]float64
+ }{
+ {
+ "validMetrics",
+ "request.time",
+ []float64{0.02, 0.09, 0.3, 0.8, 0.05},
+ map[string]float64{
+ "request.time": 0,
+ "request.time.count": 0,
+ "request.time.max": 0,
+ "request.time.median": 0,
+ "request.time.pctl95": 0,
+ },
+ map[string]float64{
+ "request.time": 0.252,
+ "request.time.count": 5,
+ "request.time.max": 0.8,
+ "request.time.median": 0.09,
+ "request.time.pctl95": 0.8,
+ },
+ },
+ {
+ "emptyMetrics",
+ "request.time",
+ []float64{},
+ map[string]float64{
+ "request.time": 0,
+ "request.time.count": 0,
+ "request.time.max": 0,
+ "request.time.median": 0,
+ "request.time.pctl95": 0,
+ },
+ map[string]float64{
+ "request.time": 0,
+ "request.time.count": 0,
+ "request.time.max": 0,
+ "request.time.median": 0,
+ "request.time.pctl95": 0,
+ },
+ },
+ {
+ "singleMetric",
+ "request.time",
+ []float64{0.07},
+ map[string]float64{
+ "request.time": 0,
+ "request.time.count": 0,
+ "request.time.max": 0,
+ "request.time.median": 0,
+ "request.time.pctl95": 0,
+ },
+ map[string]float64{
+ "request.time": 0.07,
+ "request.time.count": 1,
+ "request.time.max": 0.07,
+ "request.time.median": 0.07,
+ "request.time.pctl95": 0.07,
+ },
+ },
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(tt *testing.T) {
+ calculateTimeMetricsMap(test.metricName, test.metricTimes, test.counter)
+ assert.Equal(t, test.expectedCounter, test.counter)
+ })
+
+ }
+}
+
func TestAccessLogStats(t *testing.T) {
tests := []struct {
name string
@@ -256,6 +941,14 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.header.time.pctl95",
Value: 0,
},
+ {
+ Name: "nginx.upstream.request.count",
+ Value: 0,
+ },
+ {
+ Name: "nginx.upstream.next.count",
+ Value: 0,
+ },
{
Name: "nginx.upstream.response.length",
Value: 0,
@@ -300,6 +993,34 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.status.5xx",
Value: 0,
},
+ {
+ Name: "nginx.cache.bypass",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.expired",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.hit",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.miss",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.revalidated",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.stale",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.updating",
+ Value: 0,
+ },
},
},
},
@@ -484,6 +1205,14 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.header.time.pctl95",
Value: 0,
},
+ {
+ Name: "nginx.upstream.next.count",
+ Value: 0,
+ },
+ {
+ Name: "nginx.upstream.request.count",
+ Value: 0,
+ },
{
Name: "nginx.upstream.response.time",
Value: 0,
@@ -528,27 +1257,55 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.status.5xx",
Value: 0,
},
+ {
+ Name: "nginx.cache.bypass",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.expired",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.hit",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.miss",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.revalidated",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.stale",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.updating",
+ Value: 0,
+ },
},
},
},
{
"full_access_log_test",
- `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time" "$upstream_header_time" "$upstream_response_length" "$upstream_response_time" "$upstream_status"`,
+ `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" "$bytes_sent" "$request_length" "$request_time" "$gzip_ratio" "$server_protocol" "$upstream_connect_time" "$upstream_header_time" "$upstream_response_length" "$upstream_response_time" "$upstream_status" "$upstream_cache_status"`,
[]string{
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.00\" \"200\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350\" \"730\" \"28\" \"0.01\" \"201\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.00\" \"200\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.03\" \"400\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"100\" \"500\" \"28\" \"0.00\" \"403\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"505\" \"28\" \"0.00\" \"404\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"2000\" \"28\" \"0.00\" \"-\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"2350\" \"250\" \"28\" \"0.02\" \"500\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\" \"500\" \"28\" \"0.01\" \"502\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\" \"500\" \"28\" \"0.00\" \"503\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\" \"590\" \"28\" \"0.00\" \"502\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"900\" \"500\" \"28\" \"0.00\" \"200\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"170\" \"28\" \"0.00\" \"200\"\n",
- "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.00\" \"200\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"10\" \"HTTP/1.1\" \"350, 0.001, 0.02, -\" \"500, 0.02, -, 20\" \"28, 0, 0, 2\" \"0.00, 0.03, 0.04, -\" \"200\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"POST /nginx_status HTTP/1.1\" 201 98 \"-\" \"Go-http-client/1.1\" \"-\" \"250\" \"110\" \"0.300\" \"20\" \"HTTP/1.1\" \"350, 0.01\" \"730, 80\" \"28, 28\" \"0.01, 0.02\" \"201\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"GET /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.00\" \"200\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 400 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.03\" \"400\" \"MISS\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"DELETE /nginx_status HTTP/1.1\" 403 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"100\" \"500\" \"28\" \"0.00\" \"403\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"HEAD /nginx_status HTTP/1.1\" 404 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"505\" \"28\" \"0.00\" \"404\" \"MISS\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 499 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"350\" \"2000\" \"28\" \"0.00\" \"-\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"PUT /nginx_status HTTP/1.1\" 500 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"2350\" \"250\" \"28\" \"0.02\" \"500\" \"UPDATING\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.0\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.0\" \"350\" \"500\" \"28\" \"0.01\" \"502\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/2\" 503 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/2\" \"350\" \"500\" \"28\" \"0.00\" \"503\" \"HIT\" \n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/0.9\" 504 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/0.9\" \"350\" \"590\" \"28\" \"0.00\" \"502\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"OPTIONS /nginx_status HTTP/1.1\" 502 98 \"-\" \"Go-http-client/1.1\" \"-\" \"200\" \"100\" \"0.200\" \"-\" \"HTTP/1.1\" \"900\" \"500\" \"28\" \"0.00\" \"200\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"170\" \"28\" \"0.00\" \"200\" \"HIT\"\n",
+ "127.0.0.1 - - [19/May/2022:09:30:39 +0000] \"TRACE /nginx_status HTTP/1.1\" 200 98 \"-\" \"Go-http-client/1.1\" \"-\" \"150\" \"105\" \"0.100\" \"-\" \"HTTP/1.1\" \"350\" \"500\" \"28\" \"0.00\" \"200\" \"HIT\"\n",
},
&proto.StatsEntity{
Simplemetrics: []*proto.SimpleMetric{
@@ -686,11 +1443,11 @@ func TestAccessLogStats(t *testing.T) {
},
{
Name: "nginx.upstream.connect.time",
- Value: 514.2857142857143,
+ Value: 423.53123529411766,
},
{
Name: "nginx.upstream.connect.time.count",
- Value: 14,
+ Value: 17,
},
{
Name: "nginx.upstream.connect.time.max",
@@ -706,11 +1463,11 @@ func TestAccessLogStats(t *testing.T) {
},
{
Name: "nginx.upstream.header.time",
- Value: 588.9285714285714,
+ Value: 490.88352941176475,
},
{
Name: "nginx.upstream.header.time.count",
- Value: 14,
+ Value: 17,
},
{
Name: "nginx.upstream.header.time.max",
@@ -724,17 +1481,25 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.header.time.pctl95",
Value: 730,
},
+ {
+ Name: "nginx.upstream.request.count",
+ Value: 14,
+ },
+ {
+ Name: "nginx.upstream.next.count",
+ Value: 4,
+ },
{
Name: "nginx.upstream.response.time",
- Value: 0.005,
+ Value: 0.009411764705882354,
},
{
Name: "nginx.upstream.response.time.count",
- Value: 14,
+ Value: 17,
},
{
Name: "nginx.upstream.response.time.max",
- Value: 0.03,
+ Value: 0.04,
},
{
Name: "nginx.upstream.response.time.median",
@@ -742,11 +1507,11 @@ func TestAccessLogStats(t *testing.T) {
},
{
Name: "nginx.upstream.response.time.pctl95",
- Value: 0.02,
+ Value: 0.03,
},
{
Name: "nginx.upstream.response.length",
- Value: 28,
+ Value: 23.444444444444443,
},
{
Name: "nginx.upstream.status.1xx",
@@ -768,6 +1533,34 @@ func TestAccessLogStats(t *testing.T) {
Name: "nginx.upstream.status.5xx",
Value: 4,
},
+ {
+ Name: "nginx.cache.bypass",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.expired",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.hit",
+ Value: 11,
+ },
+ {
+ Name: "nginx.cache.miss",
+ Value: 2,
+ },
+ {
+ Name: "nginx.cache.revalidated",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.stale",
+ Value: 0,
+ },
+ {
+ Name: "nginx.cache.updating",
+ Value: 1,
+ },
},
},
},
diff --git a/src/core/metrics/sources/tailer/tailer.go b/src/core/metrics/sources/tailer/tailer.go
index bc46fb3140..4827e07bce 100644
--- a/src/core/metrics/sources/tailer/tailer.go
+++ b/src/core/metrics/sources/tailer/tailer.go
@@ -45,6 +45,7 @@ type NginxAccessItem struct {
UpstreamResponseTime string `mapstructure:"upstream_response_time"`
UpstreamResponseLength string `mapstructure:"upstream_response_length"`
UpstreamStatus string `mapstructure:"upstream_status"`
+ UpstreamCacheStatus string `mapstructure:"upstream_cache_status"`
}
func NewNginxAccessItem(v map[string]string) (*NginxAccessItem, error) {
diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go
index e4272936e9..1d5823b726 100644
--- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go
+++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go
@@ -203,8 +203,17 @@ func GetCalculationMap() map[string]string {
"nginx.http.conn.current": "avg",
"nginx.http.conn.dropped": "sum",
"nginx.http.conn.idle": "avg",
+ "nginx.cache.bypass": "sum",
+ "nginx.cache.expired": "sum",
+ "nginx.cache.hit": "sum",
+ "nginx.cache.miss": "sum",
+ "nginx.cache.revalidated": "sum",
+ "nginx.cache.stale": "sum",
+ "nginx.cache.updating": "sum",
"nginx.upstream.response.buffered": "sum",
"nginx.upstream.request.failed": "sum",
+ "nginx.upstream.request.count": "sum",
+ "nginx.upstream.next.count": "sum",
"nginx.upstream.response.failed": "sum",
"nginx.workers.count": "avg",
"nginx.workers.rlimit_nofile": "avg",
diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go
index 3f03661500..4bcd71b790 100644
--- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go
+++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go
@@ -30,6 +30,32 @@ const (
pattern = `[A-Z]+\s.+\s[A-Z]+/.+`
)
+var logVarMap = map[string]string{
+ "$remote_addr": "%{IPORHOST:remote_addr}",
+ "$remote_user": "%{USERNAME:remote_user}",
+ "$time_local": `%{HTTPDATE:time_local}`,
+ "$status": "%{INT:status}",
+ "$body_bytes_sent": "%{NUMBER:body_bytes_sent}",
+ "$http_referer": "%{DATA:http_referer}",
+ "$http_user_agent": "%{DATA:http_user_agent}",
+ "$http_x_forwarded_for": "%{DATA:http_x_forwarded_for}",
+ "$bytes_sent": "%{NUMBER:bytes_sent}",
+ "$gzip_ratio": "%{DATA:gzip_ratio}",
+ "$server_protocol": "%{DATA:server_protocol}",
+ "$request_length": "%{INT:request_length}",
+ "$request_time": "%{DATA:request_time}",
+ "\"$request\"": "\"%{DATA:request}\"",
+ "$request ": "%{DATA:request} ",
+ "$upstream_connect_time": "%{DATA:upstream_connect_time}",
+ "$upstream_header_time": "%{DATA:upstream_header_time}",
+ "$upstream_response_time": "%{DATA:upstream_response_time}",
+ "$upstream_response_length": "%{DATA:upstream_response_length}",
+ "$upstream_status": "%{DATA:upstream_status}",
+ "$upstream_cache_status": "%{DATA:upstream_cache_status}",
+ "[": "\\[",
+ "]": "\\]",
+}
+
// This metrics source is used to tail the NGINX access logs to retrieve metrics.
type NginxAccessLog struct {
@@ -156,11 +182,16 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St
}
func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) {
- logPattern := convertLogFormat(logFormat)
+ logPattern := logFormat
+
+ for key, value := range logVarMap {
+ logPattern = strings.ReplaceAll(logPattern, key, value)
+ }
+
log.Debugf("Collecting from: %s using format: %s", logFile, logFormat)
log.Debugf("Pattern used for tailing logs: %s", logPattern)
- httpCounters, upstreamCounters := getDefaultCounters()
+ httpCounters, upstreamCounters, upstreamCacheCounters := getDefaultCounters()
gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes := []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}
mu := sync.Mutex{}
@@ -180,53 +211,31 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
select {
case d := <-data:
access, err := tailer.NewNginxAccessItem(d)
+ upstreamRequest := false
if err != nil {
c.logger.Log(fmt.Sprintf("Error decoding access log entry, %v", err))
continue
}
mu.Lock()
- if access.BodyBytesSent != "" {
- if v, err := strconv.Atoi(access.BodyBytesSent); err == nil {
- n := "request.body_bytes_sent"
- httpCounters[n] = float64(v) + httpCounters[n]
- } else {
- c.logger.Log(fmt.Sprintf("Error getting body_bytes_sent value from access logs, %v", err))
- }
- }
- if access.BytesSent != "" {
- if v, err := strconv.Atoi(access.BytesSent); err == nil {
- n := "request.bytes_sent"
- httpCounters[n] = float64(v) + httpCounters[n]
- } else {
- c.logger.Log(fmt.Sprintf("Error getting bytes_sent value from access logs, %v", err))
- }
- }
+ httpCounters = c.parseAccessLogFloatCounters("request.body_bytes_sent", access.BodyBytesSent, httpCounters)
- if access.GzipRatio != "-" && access.GzipRatio != "" {
- if v, err := strconv.Atoi(access.GzipRatio); err == nil {
- gzipRatios = append(gzipRatios, float64(v))
- } else {
- c.logger.Log(fmt.Sprintf("Error getting gzip_ratio value from access logs, %v", err))
- }
- }
+ httpCounters = c.parseAccessLogFloatCounters("request.bytes_sent", access.BytesSent, httpCounters)
- if access.RequestLength != "" {
- if v, err := strconv.Atoi(access.RequestLength); err == nil {
- requestLengths = append(requestLengths, float64(v))
- } else {
- c.logger.Log(fmt.Sprintf("Error getting request_length value from access logs, %v", err))
- }
- }
+ gzipRatios = c.parseAccessLogFloatTimes("gzip_ratio", access.GzipRatio, gzipRatios)
- if access.RequestTime != "" {
- if v, err := strconv.ParseFloat(access.RequestTime, 64); err == nil {
- requestTimes = append(requestTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting request_time value from access logs, %v", err))
- }
- }
+ requestLengths = c.parseAccessLogFloatTimes("request_length", access.RequestLength, requestLengths)
+
+ requestTimes = c.parseAccessLogFloatTimes("request_time", access.RequestTime, requestTimes)
+
+ upstreamConnectTimes = c.parseAccessLogUpstream("upstream_connect_time", access.UpstreamConnectTime, upstreamConnectTimes)
+
+ upstreamHeaderTimes = c.parseAccessLogUpstream("upstream_header_time", access.UpstreamHeaderTime, upstreamHeaderTimes)
+
+ upstreamResponseLength = c.parseAccessLogUpstream("upstream_response_length", access.UpstreamResponseLength, upstreamResponseLength)
+
+ upstreamResponseTimes = c.parseAccessLogUpstream("upstream_response_time", access.UpstreamResponseTime, upstreamResponseTimes)
if access.Request != "" {
method, _, protocol := getParsedRequest(access.Request)
@@ -237,86 +246,47 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
httpCounters[n] = httpCounters[n] + 1
if access.ServerProtocol == "" {
- if strings.Count(protocol, "/") == 1 {
- httpProtocolVersion := strings.Split(protocol, "/")[1]
- httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
- n = fmt.Sprintf("v%s", httpProtocolVersion)
- httpCounters[n] = httpCounters[n] + 1
- }
- }
- }
-
- if access.UpstreamConnectTime != "-" && access.UpstreamConnectTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamConnectTime, 64); err == nil {
- upstreamConnectTimes = append(upstreamConnectTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_connect_time value from access logs, %v", err))
+ calculateServerProtocol(protocol, httpCounters)
}
}
- if access.UpstreamHeaderTime != "-" && access.UpstreamHeaderTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamHeaderTime, 64); err == nil {
- upstreamHeaderTimes = append(upstreamHeaderTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_header_time value from access logs, %v", err))
- }
+ if access.ServerProtocol != "" {
+ calculateServerProtocol(access.ServerProtocol, httpCounters)
}
- if access.UpstreamResponseLength != "-" && access.UpstreamResponseLength != "" {
- if v, err := strconv.ParseFloat(access.UpstreamResponseLength, 64); err == nil {
- upstreamResponseLength = append(upstreamResponseLength, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_response_length value from access logs, %v", err))
+ if access.UpstreamStatus != "" && access.UpstreamStatus != "-" {
+ upstreamRequest = true
+ statusValues := strings.Split(access.UpstreamStatus, ",")
+ for _, value := range statusValues {
+ if v, err := strconv.Atoi(value); err == nil {
+ n := fmt.Sprintf("upstream.status.%dxx", v/100)
+ upstreamCounters[n] = upstreamCounters[n] + 1
+ } else {
+ log.Debugf("Error getting upstream status value from access logs, %v", err)
+ }
}
}
- if access.UpstreamResponseTime != "-" && access.UpstreamResponseTime != "" {
- if v, err := strconv.ParseFloat(access.UpstreamResponseTime, 64); err == nil {
- upstreamResponseTimes = append(upstreamResponseTimes, v)
- } else {
- c.logger.Log(fmt.Sprintf("Error getting upstream_response_time value from access logs, %v", err))
- }
+ if access.UpstreamCacheStatus != "" && access.UpstreamCacheStatus != "-" {
+ upstreamRequest = true
+ calculateUpstreamCacheStatus(access.UpstreamCacheStatus, upstreamCacheCounters)
}
- if access.ServerProtocol != "" {
- if strings.Count(access.ServerProtocol, "/") == 1 {
- httpProtocolVersion := strings.Split(access.ServerProtocol, "/")[1]
- httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
- n := fmt.Sprintf("v%s", httpProtocolVersion)
- httpCounters[n] = httpCounters[n] + 1
- }
+ // don't need the http status for NGINX Plus
+ if c.nginxType == OSSNginxType {
+ c.calculateHttpStatus(access.Status, httpCounters)
}
- if access.UpstreamStatus != "" && access.UpstreamStatus != "-" {
- if v, err := strconv.Atoi(access.UpstreamStatus); err == nil {
- n := fmt.Sprintf("upstream.status.%dxx", v/100)
- upstreamCounters[n] = upstreamCounters[n] + 1
- } else {
- log.Debugf("Error getting upstream status value from access logs, %v", err)
- }
+ if access.UpstreamConnectTime != "" || access.UpstreamHeaderTime != "" || access.UpstreamResponseTime != "" {
+ upstreamTimes := []string{access.UpstreamConnectTime, access.UpstreamHeaderTime, access.UpstreamResponseTime}
+ upstreamRequest, upstreamCounters = calculateUpstreamNextCount(upstreamTimes, upstreamCounters)
}
- // don't need the http status for NGINX Plus
- if c.nginxType == OSSNginxType {
- if v, err := strconv.Atoi(access.Status); err == nil {
- n := fmt.Sprintf("status.%dxx", v/100)
- httpCounters[n] = httpCounters[n] + 1
- if v == 403 || v == 404 || v == 500 || v == 502 || v == 503 || v == 504 {
- n := fmt.Sprintf("status.%d", v)
- httpCounters[n] = httpCounters[n] + 1
- }
- if v == 499 {
- n := "status.discarded"
- httpCounters[n] = httpCounters[n] + 1
- }
- if v == 400 {
- n := "request.malformed"
- httpCounters[n] = httpCounters[n] + 1
- }
- } else {
- c.logger.Log(fmt.Sprintf("Error getting status value from access logs, %v", err))
- }
+
+ if upstreamRequest == true {
+ upstreamCounters["upstream.request.count"] = upstreamCounters["upstream.request.count"] + 1
}
+
mu.Unlock()
case <-tick.C:
@@ -334,19 +304,19 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
}
if len(requestTimes) > 0 {
- getTimeMetricsMap("request.time", requestTimes, httpCounters)
+ calculateTimeMetricsMap("request.time", requestTimes, httpCounters)
}
if len(upstreamConnectTimes) > 0 {
- getTimeMetricsMap("upstream.connect.time", upstreamConnectTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.connect.time", upstreamConnectTimes, upstreamCounters)
}
if len(upstreamHeaderTimes) > 0 {
- getTimeMetricsMap("upstream.header.time", upstreamHeaderTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.header.time", upstreamHeaderTimes, upstreamCounters)
}
if len(upstreamResponseTimes) > 0 {
- getTimeMetricsMap("upstream.response.time", upstreamResponseTimes, upstreamCounters)
+ calculateTimeMetricsMap("upstream.response.time", upstreamResponseTimes, upstreamCounters)
}
if len(upstreamResponseLength) > 0 {
@@ -358,10 +328,13 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
c.group = ""
simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(upstreamCounters)...)
+ c.group = ""
+ simpleMetrics = append(simpleMetrics, c.convertSamplesToSimpleMetrics(upstreamCacheCounters)...)
+
log.Tracef("Access log metrics collected: %v", simpleMetrics)
// reset the counters
- httpCounters, upstreamCounters = getDefaultCounters()
+ httpCounters, upstreamCounters, upstreamCacheCounters = getDefaultCounters()
gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}
c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics))
@@ -379,6 +352,72 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string
}
}
+func calculateUpstreamNextCount(metricValues []string, upstreamCounters map[string]float64) (bool, map[string]float64) {
+ upstreamRequest := false
+ for _, upstreamTimes := range metricValues {
+ if upstreamTimes != "" && upstreamTimes != "-" {
+ upstreamRequest = true
+ times := strings.Split(upstreamTimes, ", ")
+ if len(times) > 1 {
+ upstreamCounters["upstream.next.count"] = upstreamCounters["upstream.next.count"] + (float64(len(times)) - 1)
+ return upstreamRequest, upstreamCounters
+ }
+ }
+ }
+ return upstreamRequest, upstreamCounters
+}
+
+func (c *NginxAccessLog) parseAccessLogFloatTimes(metricName string, metric string, counter []float64) []float64 {
+ if metric != "" && metric != "-" {
+ if v, err := strconv.ParseFloat(metric, 64); err == nil {
+ counter = append(counter, v)
+ return counter
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ return counter
+}
+
+func (c *NginxAccessLog) parseAccessLogUpstream(metricName string, metric string, counter []float64) []float64 {
+ if metric != "" && metric != "-" {
+ metricValues := strings.Split(metric, ", ")
+ for _, value := range metricValues {
+ if value != "" && value != "-" {
+ if v, err := strconv.ParseFloat(value, 64); err == nil {
+ counter = append(counter, v)
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ }
+ return counter
+
+ }
+ return counter
+}
+
+func (c *NginxAccessLog) parseAccessLogFloatCounters(metricName string, metric string, counters map[string]float64) map[string]float64 {
+ if metric != "" {
+ if v, err := strconv.ParseFloat(metric, 64); err == nil {
+ counters[metricName] = v + counters[metricName]
+ return counters
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting %s value from access logs, %v", metricName, err))
+ }
+ }
+ return counters
+}
+
+func calculateServerProtocol(protocol string, counters map[string]float64) {
+ if strings.Count(protocol, "/") == 1 {
+ httpProtocolVersion := strings.Split(protocol, "/")[1]
+ httpProtocolVersion = strings.ReplaceAll(httpProtocolVersion, ".", "_")
+ n := fmt.Sprintf("v%s", httpProtocolVersion)
+ counters[n] = counters[n] + 1
+ }
+}
+
func getParsedRequest(request string) (method string, uri string, protocol string) {
// Looking for capital letters, a space, anything, a space, capital letters, forward slash then anything.
@@ -436,7 +475,39 @@ func getAverageMetricValue(metricValues []float64) float64 {
return value
}
-func getTimeMetricsMap(metricName string, times []float64, counter map[string]float64) {
+func (c *NginxAccessLog) calculateHttpStatus(status string, counter map[string]float64) {
+
+ if v, err := strconv.Atoi(status); err == nil {
+ n := fmt.Sprintf("status.%dxx", v/100)
+ counter[n] = counter[n] + 1
+ switch v {
+ case 403, 404, 500, 502, 503, 504:
+ n := fmt.Sprintf("status.%d", v)
+ counter[n] = counter[n] + 1
+ case 499:
+ n := "status.discarded"
+ counter[n] = counter[n] + 1
+ case 400:
+ n := "request.malformed"
+ counter[n] = counter[n] + 1
+ }
+ } else {
+ c.logger.Log(fmt.Sprintf("Error getting status value from access logs, %v", err))
+ }
+}
+
+func calculateUpstreamCacheStatus(status string, counter map[string]float64) {
+
+ n := fmt.Sprintf("cache.%s", strings.ToLower(status))
+
+ switch status {
+ case "BYPASS", "EXPIRED", "HIT", "MISS", "REVALIDATED", "STALE", "UPDATING":
+ counter[n] = counter[n] + 1
+ return
+ }
+}
+
+func calculateTimeMetricsMap(metricName string, times []float64, counter map[string]float64) {
timeMetrics := map[string]float64{
metricName: 0,
@@ -456,33 +527,6 @@ func getTimeMetricsMap(metricName string, times []float64, counter map[string]fl
}
-// convertLogFormat converts log format into a pattern that can be parsed by the tailer
-func convertLogFormat(logFormat string) string {
- newLogFormat := strings.ReplaceAll(logFormat, "$remote_addr", "%{IPORHOST:remote_addr}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$remote_user", "%{USERNAME:remote_user}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$time_local", `%{HTTPDATE:time_local}`)
- newLogFormat = strings.ReplaceAll(newLogFormat, "$status", "%{INT:status}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$body_bytes_sent", "%{NUMBER:body_bytes_sent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_referer", "%{DATA:http_referer}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_user_agent", "%{DATA:http_user_agent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$http_x_forwarded_for", "%{DATA:http_x_forwarded_for}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$bytes_sent", "%{NUMBER:bytes_sent}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$gzip_ratio", "%{DATA:gzip_ratio}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$server_protocol", "%{DATA:server_protocol}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request_length", "%{INT:request_length}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request_time", "%{DATA:request_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "\"$request\"", "\"%{DATA:request}\"")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$request ", "%{DATA:request} ")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_connect_time", "%{DATA:upstream_connect_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_header_time", "%{DATA:upstream_header_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_response_time", "%{DATA:upstream_response_time}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_response_length", "%{DATA:upstream_response_length}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "$upstream_status", "%{DATA:upstream_status}")
- newLogFormat = strings.ReplaceAll(newLogFormat, "[", "\\[")
- newLogFormat = strings.ReplaceAll(newLogFormat, "]", "\\]")
- return newLogFormat
-}
-
func isOtherMethod(method string) bool {
return method != "method.post" &&
method != "method.get" &&
@@ -492,7 +536,7 @@ func isOtherMethod(method string) bool {
method != "method.options"
}
-func getDefaultCounters() (map[string]float64, map[string]float64) {
+func getDefaultCounters() (map[string]float64, map[string]float64, map[string]float64) {
httpCounters := map[string]float64{
"gzip.ratio": 0,
"method.delete": 0,
@@ -540,6 +584,8 @@ func getDefaultCounters() (map[string]float64, map[string]float64) {
"upstream.header.time.max": 0,
"upstream.header.time.median": 0,
"upstream.header.time.pctl95": 0,
+ "upstream.request.count": 0,
+ "upstream.next.count": 0,
"upstream.response.time": 0,
"upstream.response.time.count": 0,
"upstream.response.time.max": 0,
@@ -553,5 +599,15 @@ func getDefaultCounters() (map[string]float64, map[string]float64) {
"upstream.status.5xx": 0,
}
- return httpCounters, upstreamCounters
+ upstreamCacheCounters := map[string]float64{
+ "cache.bypass": 0,
+ "cache.expired": 0,
+ "cache.hit": 0,
+ "cache.miss": 0,
+ "cache.revalidated": 0,
+ "cache.stale": 0,
+ "cache.updating": 0,
+ }
+
+ return httpCounters, upstreamCounters, upstreamCacheCounters
}
diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go
index bc46fb3140..4827e07bce 100644
--- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go
+++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/tailer/tailer.go
@@ -45,6 +45,7 @@ type NginxAccessItem struct {
UpstreamResponseTime string `mapstructure:"upstream_response_time"`
UpstreamResponseLength string `mapstructure:"upstream_response_length"`
UpstreamStatus string `mapstructure:"upstream_status"`
+ UpstreamCacheStatus string `mapstructure:"upstream_cache_status"`
}
func NewNginxAccessItem(v map[string]string) (*NginxAccessItem, error) {