Skip to content

pkg/cvo/sync_worker: TaskNodeComplete events during updates #435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wking wants to merge 2 commits into from
Closed

pkg/cvo/sync_worker: TaskNodeComplete events during updates #435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bad839c
pkg/cvo/sync_worker: TaskNodeComplete events during updates
wking Aug 13, 2020
c9705ab
WIP: Block event aggregation
wking Aug 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion pkg/cvo/cvo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,16 @@ func New(
kubeClient kubernetes.Interface,
exclude string,
) *Operator {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{
LRUCacheSize: 1000, // FIXME: intelligent tuning
BurstSize: 1000,
QPS: 1000,
MaxEvents: 1000,
KeyFunc: func(event *corev1.Event) (aggregateKey string, localKey string) {
aggregateKey, localKey = record.EventAggregatorByReasonFunc(event)
return fmt.Sprintf("%s%s", aggregateKey, event.Message), localKey
},
})
eventBroadcaster.StartLogging(klog.Infof)
eventBroadcaster.StartRecordingToSink(&coreclientsetv1.EventSinkImpl{Interface: kubeClient.CoreV1().Events(namespace)})

Expand Down
12 changes: 10 additions & 2 deletions pkg/cvo/sync_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ func (w *SyncWorker) syncOnce(ctx context.Context, work *SyncWork, maxWorkers in
Image: work.Desired.Image,
}
klog.V(4).Infof("Running sync %s (force=%t) on generation %d in state %s at attempt %d", versionString(desired), work.Desired.Force, work.Generation, work.State, work.Attempt)
cvoObjectRef := &corev1.ObjectReference{APIVersion: "config.openshift.io/v1", Kind: "ClusterVersion", Name: "version", Namespace: "openshift-cluster-version"}

// cache the payload until the release image changes
validPayload := w.payload
Expand All @@ -491,7 +492,6 @@ func (w *SyncWorker) syncOnce(ctx context.Context, work *SyncWork, maxWorkers in
desired = validPayload.Release
} else if validPayload == nil || !equalUpdate(configv1.Update{Image: validPayload.Release.Image}, configv1.Update{Image: desired.Image}) {
klog.V(4).Infof("Loading payload")
cvoObjectRef := &corev1.ObjectReference{APIVersion: "config.openshift.io/v1", Kind: "ClusterVersion", Name: "version", Namespace: "openshift-cluster-version"}
w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "RetrievePayload", "retrieving payload version=%q image=%q", desired.Version, desired.Image)
reporter.Report(SyncWorkerStatus{
Generation: work.Generation,
Expand Down Expand Up @@ -593,7 +593,11 @@ func (w *SyncWorker) syncOnce(ctx context.Context, work *SyncWork, maxWorkers in
klog.V(4).Infof("Payload loaded from %s with hash %s", desired.Image, payloadUpdate.ManifestHash)
}

return w.apply(ctx, w.payload, work, maxWorkers, reporter)
err := w.apply(ctx, w.payload, work, maxWorkers, reporter)
if err != nil && work.State == payload.UpdatingPayload {
w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeWarning, "ApplyFailed", "failed to apply update to version=%q image=%q attempt=%d: %v", work.Desired.Version, work.Desired.Image, work.Attempt, err)
}
return err
}

// apply updates the server with the contents of the provided image or returns an error.
Expand Down Expand Up @@ -704,6 +708,10 @@ func (w *SyncWorker) apply(ctx context.Context, payloadUpdate *payload.Update, w
cr.Inc()
klog.V(4).Infof("Done syncing for %s", task)
}
if work.State == payload.UpdatingPayload && len(tasks) > 0 {
cvoObjectRef := &corev1.ObjectReference{APIVersion: "config.openshift.io/v1", Kind: "ClusterVersion", Name: "version", Namespace: "openshift-cluster-version"}
w.eventRecorder.Eventf(cvoObjectRef, corev1.EventTypeNormal, "TaskNodeComplete", "synchronized task node ending in %s on update to version=%q image=%q attempt=%d", tasks[len(tasks)-1], work.Desired.Version, work.Desired.Image, work.Attempt)
}
return nil
})
if len(errs) > 0 {
Expand Down