Prevent collector deregistration on failed server conection (#242)

api/graylog.go

@@ -60,6 +60,7 @@ func RequestBackendList(httpClient *http.Client, checksum string, ctx *context.C
 		msg := "Fetching backend list"
 		system.GlobalStatus.Set(backends.StatusError, msg)
 		log.Errorf("[RequestBackendList] %s: %v", msg, err)
+		return graylog.ResponseBackendList{}, err
 	}
 
 	if resp != nil {
@@ -131,7 +132,7 @@ func RequestConfiguration(
 	return configurationResponse, nil
 }
 
-func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *graylog.StatusRequest) graylog.ResponseCollectorRegistration {
+func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *graylog.StatusRequest) (graylog.ResponseCollectorRegistration, error) {
 	c := rest.NewClient(httpClient, ctx)
 	c.BaseURL = ctx.ServerUrl
 
@@ -157,7 +158,7 @@ func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *grayl
 	r, err := c.NewRequest("PUT", "/sidecars/"+ctx.NodeId, nil, registration)
 	if err != nil {
 		log.Error("[UpdateRegistration] Can not initialize REST request")
-		return graylog.ResponseCollectorRegistration{}
+		return graylog.ResponseCollectorRegistration{}, err
 	}
 
 	respBody := new(graylog.ResponseCollectorRegistration)
@@ -167,8 +168,10 @@ func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *grayl
 		ctx.UserConfig.SendStatus = false
 	} else if resp != nil && resp.StatusCode != 202 {
 		log.Error("[UpdateRegistration] Bad response from Graylog server: ", resp.Status)
+		return graylog.ResponseCollectorRegistration{}, err
 	} else if err != nil && err != io.EOF { // err is nil for GL 2.2 and EOF for 2.1 and earlier
 		log.Error("[UpdateRegistration] Failed to report collector status to server: ", err)
+		return graylog.ResponseCollectorRegistration{}, err
 	}
 
 	// Update configuration if provided
@@ -181,7 +184,7 @@ func UpdateRegistration(httpClient *http.Client, ctx *context.Ctx, status *grayl
 		daemon.HandleCollectorActions(respBody.CollectorActions)
 	}
 
-	return *respBody
+	return *respBody, nil
 }
 
 func updateRuntimeConfiguration(respBody *graylog.ResponseCollectorRegistration, ctx *context.Ctx) error {

services/periodicals.go

@@ -39,13 +39,20 @@ func StartPeriodicals(context *context.Ctx) {
 
 	go func() {
 		backendChecksum, configurationChecksum := "", ""
+		var err error
 		for {
 			time.Sleep(time.Duration(context.UserConfig.UpdateInterval) * time.Second)
 
 			// backend list is needed before configuration assignments are fetched
-			backendChecksum = fetchBackendList(httpClient, backendChecksum, context)
+			backendChecksum, err = fetchBackendList(httpClient, backendChecksum, context)
+			if err != nil {
+				continue
+			}
 			// registration response contains configuration assignments
-			response := updateCollectorRegistration(httpClient, context)
+			response, err := updateCollectorRegistration(httpClient, context)
+			if err != nil {
+				continue
+			}
 			assignments.Store.Update(response.Assignments)
 			// create process instances
 			daemon.Daemon.SyncWithAssignments(context)
@@ -56,20 +63,20 @@ func StartPeriodicals(context *context.Ctx) {
 }
 
 // report collector status to Graylog server
-func updateCollectorRegistration(httpClient *http.Client, context *context.Ctx) graylog.ResponseCollectorRegistration {
+func updateCollectorRegistration(httpClient *http.Client, context *context.Ctx) (graylog.ResponseCollectorRegistration, error) {
 	statusRequest := api.NewStatusRequest()
 	return api.UpdateRegistration(httpClient, context, &statusRequest)
 }
 
-func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx) string {
+func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx) (string, error) {
 	response, err := api.RequestBackendList(httpClient, checksum, ctx)
 	if err != nil {
-		log.Error("Can't fetch configuration from Graylog API: ", err)
-		return ""
+		log.Error("Can't fetch collector list from Graylog API: ", err)
+		return "", err
 	}
 	if response.IsEmpty() {
 		// etag match, skipping all other actions
-		return response.Checksum
+		return response.Checksum, nil
 	}
 
 	backendList := []backends.Backend{}
@@ -78,7 +85,7 @@ func fetchBackendList(httpClient *http.Client, checksum string, ctx *context.Ctx
 	}
 	backends.Store.Update(backendList)
 
-	return response.Checksum
+	return response.Checksum, nil
 }
 
 // fetch configuration periodically