Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extend Upstream Servers with pod_name label #1107

Merged
merged 2 commits into from
Aug 28, 2020
Merged

Extend Upstream Servers with pod_name label #1107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
09e9f7c
Extend Upstream Servers with pod_name label
LorcanMcVeigh Aug 27, 2020
6729361
Update vendor
LorcanMcVeigh Aug 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/nginx-ingress/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,8 @@ func main() {
if *nginxPlus {
upstreamServerVariableLabels := []string{"service", "resource_type", "resource_name", "resource_namespace"}
serverZoneVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"}
variableLabelNames := nginxCollector.NewVariableLabelNames(upstreamServerVariableLabels, serverZoneVariableLabels)
upstreamServerPeerVariableLabelNames := []string{"pod_name"}
variableLabelNames := nginxCollector.NewVariableLabelNames(upstreamServerVariableLabels, serverZoneVariableLabels, upstreamServerPeerVariableLabelNames)
plusCollector = nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus", variableLabelNames, constLabels)
go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusCollector, registry)
} else {
Expand Down
10 changes: 6 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,16 @@ require (
github.com/googleapis/gnostic v0.3.1 // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
github.com/imdario/mergo v0.3.8 // indirect
github.com/nginxinc/nginx-plus-go-client v0.6.0
github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20200804105119-66a9427a4093
github.com/prometheus/client_golang v1.6.0
github.com/nginxinc/nginx-plus-go-client v0.7.0
github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20200825091536-6b1901af5792
github.com/prometheus/client_golang v1.7.1
github.com/prometheus/common v0.13.0 // indirect
github.com/spiffe/go-spiffe v1.0.0
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6 // indirect
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1 // indirect
google.golang.org/appengine v1.6.5 // indirect
google.golang.org/protobuf v1.25.0 // indirect
k8s.io/api v0.18.0
k8s.io/apimachinery v0.18.1-beta.0
k8s.io/client-go v0.18.0
Expand Down
234 changes: 234 additions & 0 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

90 changes: 64 additions & 26 deletions internal/configs/configurator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ type tlsPassthroughPair struct {
// metricLabelsIndex keeps the relations between Ingress Controller resources and NGINX configuration.
// Used to be able to add Prometheus Metrics variable labels grouped by resource key.
type metricLabelsIndex struct {
ingressUpstreams map[string][]string
virtualServerUpstreams map[string][]string
ingressServerZones map[string][]string
virtualServerServerZones map[string][]string
ingressUpstreams map[string][]string
virtualServerUpstreams map[string][]string
ingressServerZones map[string][]string
virtualServerServerZones map[string][]string
ingressUpstreamPeers map[string][]string
virtualServerUpstreamPeers map[string][]string
}

// Configurator configures NGINX.
Expand All @@ -80,21 +82,23 @@ type Configurator struct {
tlsPassthroughPairs map[string]tlsPassthroughPair
isWildcardEnabled bool
isPlus bool
nginxPlusCollector *collector.NginxPlusCollector
labelUpdater collector.LabelUpdater
metricLabelsIndex *metricLabelsIndex
isPrometheusEnabled bool
}

// NewConfigurator creates a new Configurator.
func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigParams, config *ConfigParams, globalCfgParams *GlobalConfigParams,
templateExecutor *version1.TemplateExecutor, templateExecutorV2 *version2.TemplateExecutor, isPlus bool, isWildcardEnabled bool,
plusCollector *collector.NginxPlusCollector, isPrometheusEnabled bool) *Configurator {
labelUpdater collector.LabelUpdater, isPrometheusEnabled bool) *Configurator {

metricLabelsIndex := &metricLabelsIndex{
ingressUpstreams: make(map[string][]string),
virtualServerUpstreams: make(map[string][]string),
ingressServerZones: make(map[string][]string),
virtualServerServerZones: make(map[string][]string),
ingressUpstreams: make(map[string][]string),
virtualServerUpstreams: make(map[string][]string),
ingressServerZones: make(map[string][]string),
virtualServerServerZones: make(map[string][]string),
ingressUpstreamPeers: make(map[string][]string),
virtualServerUpstreamPeers: make(map[string][]string),
}

cnf := Configurator{
Expand All @@ -110,7 +114,7 @@ func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigPa
tlsPassthroughPairs: make(map[string]tlsPassthroughPair),
isPlus: isPlus,
isWildcardEnabled: isWildcardEnabled,
nginxPlusCollector: plusCollector,
labelUpdater: labelUpdater,
metricLabelsIndex: metricLabelsIndex,
isPrometheusEnabled: isPrometheusEnabled,
}
Expand Down Expand Up @@ -145,18 +149,34 @@ func (cnf *Configurator) updateIngressMetricsLabels(ingEx *IngressEx, upstreams
newUpstreams := make(map[string]bool)
var newUpstreamsNames []string

upstreamServerPeerLabels := make(map[string][]string)
newPeers := make(map[string]bool)
var newPeersIPs []string

for _, u := range upstreams {
upstreamServerLabels[u.Name] = createUpstreamServerLabels(u.UpstreamLabels.Service, u.UpstreamLabels.ResourceType, u.UpstreamLabels.ResourceName, u.UpstreamLabels.ResourceNamespace)
newUpstreams[u.Name] = true
newUpstreamsNames = append(newUpstreamsNames, u.Name)
for _, server := range u.UpstreamServers {
s := fmt.Sprintf("%v:%v", server.Address, server.Port)
podName := ingEx.PodsByIP[s]
labelKey := fmt.Sprintf("%v/%v", u.Name, s)
upstreamServerPeerLabels[labelKey] = []string{podName}
newPeers[labelKey] = true
newPeersIPs = append(newPeersIPs, labelKey)
}
}

key := fmt.Sprintf("%v/%v", ingEx.Ingress.Namespace, ingEx.Ingress.Name)

removedUpstreams := findRemovedKeys(cnf.metricLabelsIndex.ingressUpstreams[key], newUpstreams)
cnf.metricLabelsIndex.ingressUpstreams[key] = newUpstreamsNames
cnf.nginxPlusCollector.UpdateUpstreamServerLabels(upstreamServerLabels)
cnf.nginxPlusCollector.DeleteUpstreamServerLabels(removedUpstreams)
cnf.labelUpdater.UpdateUpstreamServerLabels(upstreamServerLabels)
cnf.labelUpdater.DeleteUpstreamServerLabels(removedUpstreams)

removedPeers := findRemovedKeys(cnf.metricLabelsIndex.ingressUpstreamPeers[key], newPeers)
cnf.metricLabelsIndex.ingressUpstreamPeers[key] = newPeersIPs
cnf.labelUpdater.UpdateUpstreamServerPeerLabels(upstreamServerPeerLabels)
cnf.labelUpdater.DeleteUpstreamServerPeerLabels(removedPeers)

serverZoneLabels := make(map[string][]string)
newZones := make(map[string]bool)
Expand All @@ -169,15 +189,17 @@ func (cnf *Configurator) updateIngressMetricsLabels(ingEx *IngressEx, upstreams

removedZones := findRemovedKeys(cnf.metricLabelsIndex.ingressServerZones[key], newZones)
cnf.metricLabelsIndex.ingressServerZones[key] = newZonesNames
cnf.nginxPlusCollector.UpdateServerZoneLabels(serverZoneLabels)
cnf.nginxPlusCollector.DeleteServerZoneLabels(removedZones)
cnf.labelUpdater.UpdateServerZoneLabels(serverZoneLabels)
cnf.labelUpdater.DeleteServerZoneLabels(removedZones)
}

func (cnf *Configurator) deleteIngressMetricsLabels(key string) {
cnf.nginxPlusCollector.DeleteUpstreamServerLabels(cnf.metricLabelsIndex.ingressUpstreams[key])
cnf.nginxPlusCollector.DeleteServerZoneLabels(cnf.metricLabelsIndex.ingressServerZones[key])
cnf.labelUpdater.DeleteUpstreamServerLabels(cnf.metricLabelsIndex.ingressUpstreams[key])
cnf.labelUpdater.DeleteServerZoneLabels(cnf.metricLabelsIndex.ingressServerZones[key])
cnf.labelUpdater.DeleteUpstreamServerPeerLabels(cnf.metricLabelsIndex.ingressUpstreamPeers[key])
delete(cnf.metricLabelsIndex.ingressUpstreams, key)
delete(cnf.metricLabelsIndex.ingressServerZones, key)
delete(cnf.metricLabelsIndex.ingressUpstreamPeers, key)
}

// AddOrUpdateIngress adds or updates NGINX configuration for the Ingress resource.
Expand Down Expand Up @@ -208,7 +230,6 @@ func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) error {
cnf.nginxManager.CreateConfig(name, content)

cnf.ingresses[name] = ingEx

if cnf.isPlus && cnf.isPrometheusEnabled {
cnf.updateIngressMetricsLabels(ingEx, nginxCfg.Upstreams)
}
Expand Down Expand Up @@ -254,7 +275,6 @@ func (cnf *Configurator) addOrUpdateMergeableIngress(mergeableIngs *MergeableIng
minionName := objectMetaToFileName(&minion.Ingress.ObjectMeta)
cnf.minions[name][minionName] = true
}

if cnf.isPlus && cnf.isPrometheusEnabled {
cnf.updateIngressMetricsLabels(mergeableIngs.Master, nginxCfg.Upstreams)
}
Expand All @@ -267,18 +287,34 @@ func (cnf *Configurator) updateVirtualServerMetricsLabels(virtualServerEx *Virtu
newUpstreams := make(map[string]bool)
var newUpstreamsNames []string

upstreamServerPeerLabels := make(map[string][]string)
newPeers := make(map[string]bool)
var newPeersIPs []string

for _, u := range upstreams {
labels[u.Name] = createUpstreamServerLabels(u.UpstreamLabels.Service, u.UpstreamLabels.ResourceType, u.UpstreamLabels.ResourceName, u.UpstreamLabels.ResourceNamespace)
newUpstreams[u.Name] = true
newUpstreamsNames = append(newUpstreamsNames, u.Name)
for _, server := range u.Servers {
podName := virtualServerEx.PodsByIP[server.Address]
labelKey := fmt.Sprintf("%v/%v", u.Name, server.Address)
upstreamServerPeerLabels[labelKey] = []string{podName}
newPeers[labelKey] = true
newPeersIPs = append(newPeersIPs, labelKey)
}
}

key := fmt.Sprintf("%v/%v", virtualServerEx.VirtualServer.Namespace, virtualServerEx.VirtualServer.Name)

removedPeers := findRemovedKeys(cnf.metricLabelsIndex.virtualServerUpstreamPeers[key], newPeers)
cnf.metricLabelsIndex.virtualServerUpstreamPeers[key] = newPeersIPs
cnf.labelUpdater.UpdateUpstreamServerPeerLabels(upstreamServerPeerLabels)
cnf.labelUpdater.DeleteUpstreamServerPeerLabels(removedPeers)

removedUpstreams := findRemovedKeys(cnf.metricLabelsIndex.virtualServerUpstreams[key], newUpstreams)
cnf.nginxPlusCollector.UpdateUpstreamServerLabels(labels)
cnf.labelUpdater.UpdateUpstreamServerLabels(labels)
cnf.metricLabelsIndex.virtualServerUpstreams[key] = newUpstreamsNames
cnf.nginxPlusCollector.DeleteUpstreamServerLabels(removedUpstreams)
cnf.labelUpdater.DeleteUpstreamServerLabels(removedUpstreams)

serverZoneLabels := make(map[string][]string)
newZones := make(map[string]bool)
Expand All @@ -291,15 +327,17 @@ func (cnf *Configurator) updateVirtualServerMetricsLabels(virtualServerEx *Virtu

removedZones := findRemovedKeys(cnf.metricLabelsIndex.virtualServerServerZones[key], newZones)
cnf.metricLabelsIndex.virtualServerServerZones[key] = newZonesNames
cnf.nginxPlusCollector.UpdateServerZoneLabels(serverZoneLabels)
cnf.nginxPlusCollector.DeleteServerZoneLabels(removedZones)
cnf.labelUpdater.UpdateServerZoneLabels(serverZoneLabels)
cnf.labelUpdater.DeleteServerZoneLabels(removedZones)
}

func (cnf *Configurator) deleteVirtualServerMetricsLabels(key string) {
cnf.nginxPlusCollector.DeleteUpstreamServerLabels(cnf.metricLabelsIndex.virtualServerUpstreams[key])
cnf.nginxPlusCollector.DeleteServerZoneLabels(cnf.metricLabelsIndex.virtualServerServerZones[key])
cnf.labelUpdater.DeleteUpstreamServerLabels(cnf.metricLabelsIndex.virtualServerUpstreams[key])
cnf.labelUpdater.DeleteServerZoneLabels(cnf.metricLabelsIndex.virtualServerServerZones[key])
cnf.labelUpdater.DeleteUpstreamServerPeerLabels(cnf.metricLabelsIndex.virtualServerUpstreamPeers[key])
delete(cnf.metricLabelsIndex.virtualServerUpstreams, key)
delete(cnf.metricLabelsIndex.virtualServerServerZones, key)
delete(cnf.metricLabelsIndex.virtualServerUpstreamPeers, key)
}

// AddOrUpdateVirtualServer adds or updates NGINX configuration for the VirtualServer resource.
Expand Down
Loading