Monitor NGINX logs for errors & NGINX worker processes after a NGINX reload

Makefile

@@ -97,7 +97,6 @@ build/nginx-agent:
 build: build/nginx-agent ## Build agent executable
 
 deps: ## Update dependencies in vendor folders
-	git diff --quiet || { echo "Local changes found. Please commit or stash your changes." >&2; exit 1; }
 	cd sdk && make generate
 	for dir in ${VENDOR_LOCATIONS}; do \
 		(cd "$$dir" && echo "Running vendor commands on $$dir" && go mod tidy && go mod vendor && cd "$$OLDPWD" || exit) \
@@ -264,14 +263,14 @@ image: ## Build agent container image for NGINX Plus, need nginx-repo.crt and ng
 oss-image: ## Build agent container image for NGINX OSS
 	@echo Building image with $(CONTAINER_CLITOOL); \
 	$(CONTAINER_BUILDENV) $(CONTAINER_CLITOOL) build -t ${IMAGE_TAG} . \
-	--no-cache -f ./scripts/docker/nginx-oss/${OS_RELEASE}/Dockerfile \
+	--no-cache -f ./scripts/docker/nginx-oss/${CONTAINER_OS_TYPE}/Dockerfile \
 	--build-arg PACKAGE_NAME=${PACKAGE_NAME} \
 	--build-arg BASE_IMAGE=${BASE_IMAGE} \
-	--build-arg ENTRY_POINT=./scripts/docker/nginx-oss/entrypoint.sh
+	--build-arg ENTRY_POINT=./scripts/docker/entrypoint.sh
 
 run-container: ## Run container from specified IMAGE_TAG
 	@echo Running ${IMAGE_TAG} with $(CONTAINER_CLITOOL); \
-		$(CONTAINER_CLITOOL) run --mount type=bind,source=${PWD}/nginx-agent.conf,target=/etc/nginx-agent/nginx-agent.conf ${IMAGE_TAG}
+		$(CONTAINER_CLITOOL) run -p 127.0.0.1:8081:8081/tcp --mount type=bind,source=${PWD}/nginx-agent.conf,target=/etc/nginx-agent/nginx-agent.conf ${IMAGE_TAG}
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # Grafana Example Dashboard Targets                                                                               #

go.mod

@@ -39,7 +39,7 @@ require (
 
 require (
 	github.com/bufbuild/buf v1.15.1
-	github.com/evilmartians/lefthook v1.3.7
+	github.com/evilmartians/lefthook v1.3.8
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/go-swagger/go-swagger v0.30.4
 	github.com/goreleaser/nfpm/v2 v2.27.1

go.sum

@@ -149,8 +149,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.3.0-java h1:bV5JGEB1ouEzZa0hgVDFFiClrUEuGWRaAc/3mxR2QK0=
 github.com/envoyproxy/protoc-gen-validate v0.3.0-java/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evilmartians/lefthook v1.3.7 h1:h2lvNl5OSlnAZdV3bdA283itH6pHKaTnzk837dQqrCk=
-github.com/evilmartians/lefthook v1.3.7/go.mod h1:FTw9v/+5Tg5AlrLPxS2kJeqHhQNqhUA+6/WdGeaf41s=
+github.com/evilmartians/lefthook v1.3.8 h1:5SOqN6vd7S7FYvr9MDNM2g2lq/kTE3uTWY7Qt63uuj8=
+github.com/evilmartians/lefthook v1.3.8/go.mod h1:FTw9v/+5Tg5AlrLPxS2kJeqHhQNqhUA+6/WdGeaf41s=
 github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
 github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=

sdk/proto/events/event.proto

@@ -34,17 +34,17 @@ message Event {
     ActivityEvent ActivityEvent = 2 [(gogoproto.jsontag) = "activity_event"]; // Activity event
 
     /**
-       * While generating the SecurityViolationEvent, the Metadata for a SecurityViolationEvent
-       * would look as shown below:
-       *  -  Module           = Agent
-       *  -  UUID             = A UUID generated by the Agent for the EventReport
-       *  -  CorrelationID    = The UUID will be used as the Correlation ID to track the EventReport
-       *  -  Timestamp        = The timestamp when NGINX Agent received the violation event
-       *  -  EventLevel       = All the SecurityViolationEvent would be generated at an ERROR_EVENT_LEVEL ("ERROR") level
-       *                        In future, the levels might be dynamically chosen based on Request Outcome of SecurityViolationEvent
-       *  -  Type             = NGINX_EVENT_TYPE ("Nginx")
-       *  -  Category         = APP_PROTECT_CATEGORY ("AppProtect")
-    */
+     * While generating the SecurityViolationEvent, the Metadata for a SecurityViolationEvent
+     * would look as shown below:
+     *  -  Module           = Agent
+     *  -  UUID             = A UUID generated by the Agent for the EventReport
+     *  -  CorrelationID    = The UUID will be used as the Correlation ID to track the EventReport
+     *  -  Timestamp        = The timestamp when NGINX Agent received the violation event
+     *  -  EventLevel       = All the SecurityViolationEvent would be generated at an ERROR_EVENT_LEVEL ("ERROR") level
+     *                        In future, the levels might be dynamically chosen based on Request Outcome of SecurityViolationEvent
+     *  -  Type             = NGINX_EVENT_TYPE ("Nginx")
+     *  -  Category         = APP_PROTECT_CATEGORY ("AppProtect")
+     */
 
     SecurityViolationEvent SecurityViolationEvent = 3 [(gogoproto.jsontag) = "security_violation_event"]; // Security violation event
   }

src/core/config/config.go

@@ -78,6 +78,7 @@ func SetDefaults() {
 
 	// NGINX DEFAULTS
 	Viper.SetDefault(NginxClientVersion, Defaults.Nginx.NginxClientVersion)
+	Viper.SetDefault(NginxConfigReloadMonitoringPeriod, Defaults.Nginx.ConfigReloadMonitoringPeriod)
 }
 
 func setFlagDeprecated(name string, usageMessage string) {
@@ -284,10 +285,11 @@ func getDataplane() Dataplane {
 
 func getNginx() Nginx {
 	return Nginx{
-		ExcludeLogs:         Viper.GetString(NginxExcludeLogs),
-		Debug:               Viper.GetBool(NginxDebug),
-		NginxCountingSocket: Viper.GetString(NginxCountingSocket),
-		NginxClientVersion:  Viper.GetInt(NginxClientVersion),
+		ExcludeLogs:                  Viper.GetString(NginxExcludeLogs),
+		Debug:                        Viper.GetBool(NginxDebug),
+		NginxCountingSocket:          Viper.GetString(NginxCountingSocket),
+		NginxClientVersion:           Viper.GetInt(NginxClientVersion),
+		ConfigReloadMonitoringPeriod: Viper.GetDuration(NginxConfigReloadMonitoringPeriod),
 	}
 }
 

src/core/config/config_test.go

@@ -148,6 +148,7 @@ func TestGetConfig(t *testing.T) {
 		assert.Equal(t, Defaults.Server.Metrics, config.Server.Metrics)
 
 		assert.Equal(t, Defaults.AgentAPI.Port, config.AgentAPI.Port)
+		assert.Equal(t, Defaults.Nginx.ConfigReloadMonitoringPeriod, config.Nginx.ConfigReloadMonitoringPeriod)
 
 		assert.True(t, len(config.AllowedDirectoriesMap) > 0)
 		assert.Equal(t, Defaults.ConfigDirs, config.ConfigDirs)

src/core/config/defaults.go

@@ -51,9 +51,10 @@ var (
 			Token: uuid.New().String(),
 		},
 		Nginx: Nginx{
-			Debug:               false,
-			NginxCountingSocket: "unix:/var/run/nginx-agent/nginx.sock",
-			NginxClientVersion:  6,
+			Debug:                        false,
+			NginxCountingSocket:          "unix:/var/run/nginx-agent/nginx.sock",
+			NginxClientVersion:           6,
+			ConfigReloadMonitoringPeriod: 10 * time.Second,
 		},
 		ConfigDirs:            "/etc/nginx:/usr/local/etc/nginx:/usr/share/nginx/modules:/etc/nms",
 		AllowedDirectoriesMap: map[string]struct{}{},
@@ -128,10 +129,11 @@ const (
 	// viper keys used in config
 	NginxKey = "nginx"
 
-	NginxExcludeLogs    = NginxKey + agent_config.KeyDelimiter + "exclude_logs"
-	NginxDebug          = NginxKey + agent_config.KeyDelimiter + "debug"
-	NginxCountingSocket = NginxKey + agent_config.KeyDelimiter + "socket"
-	NginxClientVersion  = NginxKey + agent_config.KeyDelimiter + "client_version"
+	NginxExcludeLogs                  = NginxKey + agent_config.KeyDelimiter + "exclude_logs"
+	NginxDebug                        = NginxKey + agent_config.KeyDelimiter + "debug"
+	NginxCountingSocket               = NginxKey + agent_config.KeyDelimiter + "socket"
+	NginxClientVersion                = NginxKey + agent_config.KeyDelimiter + "client_version"
+	NginxConfigReloadMonitoringPeriod = NginxKey + agent_config.KeyDelimiter + "config_reload_monitoring_period"
 
 	// viper keys used in config
 	DataplaneKey = "dataplane"
@@ -247,6 +249,11 @@ var (
 			Usage:        "The NGINX Plus counting unix socket location.",
 			DefaultValue: Defaults.Nginx.NginxCountingSocket,
 		},
+		&DurationFlag{
+			Name:         NginxConfigReloadMonitoringPeriod,
+			Usage:        "The duration the NGINX Agent will monitor error logs after a NGINX reload",
+			DefaultValue: Defaults.Nginx.ConfigReloadMonitoringPeriod,
+		},
 		// Metrics
 		&DurationFlag{
 			Name:         MetricsCollectionInterval,

src/core/config/types.go

@@ -88,10 +88,11 @@ type TLSConfig struct {
 
 // Nginx settings
 type Nginx struct {
-	ExcludeLogs         string `mapstructure:"exclude_logs" yaml:"-"`
-	Debug               bool   `mapstructure:"debug" yaml:"-"`
-	NginxCountingSocket string `mapstructure:"socket" yaml:"-"`
-	NginxClientVersion  int    `mapstructure:"client_version" yaml:"-"`
+	ExcludeLogs                  string        `mapstructure:"exclude_logs" yaml:"-"`
+	Debug                        bool          `mapstructure:"debug" yaml:"-"`
+	NginxCountingSocket          string        `mapstructure:"socket" yaml:"-"`
+	NginxClientVersion           int           `mapstructure:"client_version" yaml:"-"`
+	ConfigReloadMonitoringPeriod time.Duration `mapstructure:"config_reload_monitoring_period" yaml:"-"`
 }
 
 type Dataplane struct {

src/core/metrics/sources/nginx_access_log.go

@@ -20,7 +20,7 @@ import (
 	"github.com/nginx/agent/sdk/v2/proto"
 	"github.com/nginx/agent/v2/src/core"
 	"github.com/nginx/agent/v2/src/core/metrics"
-	"github.com/nginx/agent/v2/src/core/metrics/sources/tailer"
+	"github.com/nginx/agent/v2/src/core/tailer"
 
 	log "github.com/sirupsen/logrus"
 )

src/core/metrics/sources/nginx_error_log.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/nginx/agent/v2/src/core"
 	"github.com/nginx/agent/v2/src/core/metrics"
-	"github.com/nginx/agent/v2/src/core/metrics/sources/tailer"
+	"github.com/nginx/agent/v2/src/core/tailer"
 
 	log "github.com/sirupsen/logrus"
 )

src/core/nginx.go

@@ -330,8 +330,8 @@ func (n *NginxBinaryType) WriteConfig(config *proto.NginxConfig) (*sdk.ConfigApp
 		}
 	}
 
-	details, ok := n.nginxDetailsMap[config.ConfigData.NginxId]
-	if !ok || details == nil {
+	details := n.GetNginxDetailsByID(config.ConfigData.NginxId)
+	if details == nil {
 		return nil, fmt.Errorf("NGINX instance %s not found", config.ConfigData.NginxId)
 	}
 

src/core/metrics/sources/tailer/tailer.go → src/core/tailer/tailer.go

@@ -19,9 +19,10 @@ import (
 
 var (
 	tailConfig = tail.Config{
-		Follow: true,
-		ReOpen: true,
-		Poll:   true,
+		Follow:    true,
+		ReOpen:    true,
+		MustExist: true,
+		Poll:      true,
 		Location: &tail.SeekInfo{
 			Whence: io.SeekEnd,
 		},
@@ -108,11 +109,14 @@ func (t *Tailer) Tail(ctx context.Context, data chan<- string) {
 			data <- line.Text
 
 		case <-ctx.Done():
-			err := ctx.Err()
-			if err != nil {
-				log.Errorf("error in done context Tail %v", err)
+			ctxErr := ctx.Err()
+			switch ctxErr {
+			case context.DeadlineExceeded:
+				log.Tracef("Tailer cancelled. Deadline exceeded, %v", ctxErr)
+			case context.Canceled:
+				log.Tracef("Tailer forcibly cancelled, %v", ctxErr)
 			}
-			log.Info("tailer is done")
+			log.Trace("Tailer is done")
 			return
 		}
 	}
@@ -134,11 +138,14 @@ func (t *PatternTailer) Tail(ctx context.Context, data chan<- map[string]string)
 				data <- l
 			}
 		case <-ctx.Done():
-			err := ctx.Err()
-			if err != nil {
-				log.Errorf("error in done context Tail %v", err)
+			ctxErr := ctx.Err()
+			switch ctxErr {
+			case context.DeadlineExceeded:
+				log.Tracef("Tailer cancelled because deadline was exceeded, %v", ctxErr)
+			case context.Canceled:
+				log.Tracef("Tailer forcibly cancelled, %v", ctxErr)
 			}
-			log.Info("tailer is done")
+			log.Tracef("Tailer is done")
 			return
 		}
 	}