feat: add version metrics. Closes #109

.github/workflows/release.yml

@@ -34,8 +34,9 @@ jobs:
           tag=$(basename $GITHUB_REF)
           if [ $tag = "main" ]; then
             tag="latest"
+            version="v0.0.0-latest-0"
           fi
-          docker buildx build . --target ${{matrix.target}} --tag quay.io/argoproj/dataflow-${{matrix.target}}:$tag --push --build-arg VERSION="$tag"
+          docker buildx build . --target ${{matrix.target}} --tag quay.io/argoproj/dataflow-${{matrix.target}}:$tag --push --build-arg VERSION="$version"
 
   publish-release:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## v0.0.65 (2021-07-13)
+
+ * [e8fa437](https://github.com/argoproj-labs/argo-dataflow/commit/e8fa437ccd0f82ed6a2b50a194edb1b738444a22) fix: use shared counter for Kafka commit
+ * [3ad7cf1](https://github.com/argoproj-labs/argo-dataflow/commit/3ad7cf1ddcf70752ebdd9684bb8ccb10aed0d8f1) fix: stop Kafka dropping messages on disruption
+ * [9a86950](https://github.com/argoproj-labs/argo-dataflow/commit/9a8695069c1d68923e6340d75d6a420c1805a339) fix: return 503 if HTTP source not ready due to pre-stop
+ * [2155591](https://github.com/argoproj-labs/argo-dataflow/commit/2155591cbbe026a403e690a55b51cfece68e2adf) feat: change default deletion delay to 720h (~30d)
+ * [8fb4c17](https://github.com/argoproj-labs/argo-dataflow/commit/8fb4c17eb838023e43e1e647205c185745ac434b) fix!: harmonize `retries`
+ * [0aa7aa4](https://github.com/argoproj-labs/argo-dataflow/commit/0aa7aa40281d0f1844c3a253659c428b6df7e823) fix: add timeout to HTTP sink
+ * [f5e89b9](https://github.com/argoproj-labs/argo-dataflow/commit/f5e89b99ad46240b0f4b175f95476c2f9d4edec0) fix: correct hash
+
+### Contributors
+
+ * Alex Collins
+
 ## v0.0.64 (2021-07-12)
 
  * [11b2033](https://github.com/argoproj-labs/argo-dataflow/commit/11b2033b81a4ac2ecdd9e449cd3c11bba652e653) feat: update resources to only apply to built-ins

Makefile

@@ -3,6 +3,7 @@
 
 # Image URL to use all building/pushing image targets
 TAG ?= latest
+VERSION ?= v0.0.0-latest-0
 CONFIG ?= dev
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
@@ -134,10 +135,14 @@ java16: java16-image
 python3-9: python3-9-image
 
 %-image:
-	docker buildx build . --target $* --tag quay.io/argoproj/dataflow-$*:$(TAG) --load --build-arg VERSION="$(TAG)"
+	docker buildx build . --target $* --tag quay.io/argoproj/dataflow-$*:$(TAG) --load --build-arg VERSION="$(VERSION)"
 ifeq ($(K3D),true)
 	k3d image import quay.io/argoproj/dataflow-$*:$(TAG)
 endif
+
+scan: scan-controller scan-runner scan-testapi scan-go1-16 scan-java16 scan-python3-9
+	snyk test --severity-threshold=high
+
 scan-%:
 	docker scan --severity=high quay.io/argoproj/dataflow-$*:$(TAG)
 

Procfile

@@ -1,4 +1,4 @@
-controller: source manager.env && go run -race -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.version=latest'" ./manager
+controller: source manager.env && go run -race -ldflags="-X 'github.com/argoproj-labs/argo-dataflow/shared/util.version=v0.0.0-latest-0'" ./manager
 logs: make logs
 argocli: make argocli
 ui: make ui

config/ci.yaml

@@ -4847,7 +4847,7 @@ spec:
   containers:
   - args:
     - -fvN
-    - http://testapi/kafka/pump-topic?topic=input-topic&sleep=500ms
+    - http://testapi/kafka/pump-topic?topic=input-topic&sleep=2s
     command:
     - curl
     image: golang:1.16

config/dev.yaml

@@ -4846,7 +4846,7 @@ spec:
   containers:
   - args:
     - -fvN
-    - http://testapi/kafka/pump-topic?topic=input-topic&sleep=500ms
+    - http://testapi/kafka/pump-topic?topic=input-topic&sleep=2s
     command:
     - curl
     image: golang:1.16

docs/SECURITY.md

@@ -4,7 +4,7 @@
 
 * For base images prefer `scratch` then `distroless` then `alpine`.
 * Snyk is used to scan images.
-* Synk is used to scan imported Go modules.
+* Snyk is used to scan imported Go modules.
 
 ## Configuration
 

docs/VERSIONING.md

@@ -0,0 +1,6 @@
+# Versioning
+
+We use Semantic Versioning, with "v" prefix. Semantic version are not actually prefixed with "v", but basically everyone
+tolerates this.
+
+We also build the tip of the `main` branch; tag it `:latest` and version `v0.0.0-latest-0`. 

go.mod

@@ -5,7 +5,7 @@ go 1.16
 require (
 	cloud.google.com/go v0.52.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/Shopify/sarama v1.29.1
 	github.com/antonmedv/expr v1.8.9

manager/controllers/config.go

@@ -27,7 +27,11 @@ func init() {
 	if imagePrefix == "" {
 		imagePrefix = "quay.io/argoproj"
 	}
-	imageFormat = fmt.Sprintf("%s/%s:%s", imagePrefix, "%s", util.Version())
+	tag := util.Version.Original() //we don't use String() because semantic version do not have "v" prefix
+	if tag == "v0.0.0-latest-0" {
+		tag = "latest"
+	}
+	imageFormat = fmt.Sprintf("%s/%s:%s", imagePrefix, "%s", tag)
 	runnerImage = fmt.Sprintf(imageFormat, "dataflow-runner")
 	if text, ok := os.LookupEnv(dfv1.EnvUpdateInterval); ok {
 		if v, err := time.ParseDuration(text); err != nil {

runner/sidecar/sidecar.go

@@ -106,6 +106,18 @@ func Exec(ctx context.Context) error {
 			Name: "replicas",
 			Help: "Number of replicas, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#replicas",
 		}, func() float64 { return float64(step.Status.Replicas) })
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "version_major",
+			Help: "Major version number, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#version_major",
+		}, func() float64 { return float64(sharedutil.Version.Major()) })
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "version_minor",
+			Help: "Minor version number, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#version_minor",
+		}, func() float64 { return float64(sharedutil.Version.Minor()) })
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "version_patch",
+			Help: "Patch version number, see https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/METRICS.md#version_patch",
+		}, func() float64 { return float64(sharedutil.Version.Patch()) })
 	}
 
 	// we listen to this message, but it does not come from Kubernetes, it actually comes from the main container's

shared/util/version.go

@@ -1,12 +1,20 @@
 package util
 
+import (
+	"github.com/Masterminds/semver"
+)
+
 var (
-	version string
+	// The version MUST be "v"+semanticVersion, it should be one of the following options
+	// * "vX.Y.Z" for released version, e.g. "v1.2.3"
+	// * "v0.0.0-X-Y" for unreleased versions, e.g.
+	//   - "v0.0.0-latest-0" ("latest" version, i.e. latest build on the "main" branch or local dev build)
+	version = "v0.0.0-latest-0"
 	logger  = NewLogger()
+	Version semver.Version
 )
 
 func init() {
 	logger.Info("version", "version", version)
+	Version = *semver.MustParse(version)
 }
-
-func Version() string { return version }

test/e2e/metrics_test.go

@@ -43,6 +43,9 @@ func TestMetrics(t *testing.T) {
 	WaitForStep(TotalSunkMessages(1))
 
 	ExpectMetric("input_inflight", 0)
+	ExpectMetric("version_major", 0)
+	ExpectMetric("version_minor", 0)
+	ExpectMetric("version_patch", 0)
 	ExpectMetric("replicas", 1)
 	ExpectMetric("sources_errors", 0)
 	ExpectMetric("sources_pending", 0)