diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c681d66c27..0d78bcb3a3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -285,6 +285,16 @@ functions: echo "Response Body: $response_body" echo "HTTP Status: $http_status" + send-perf-pr-comment: + - command: subprocess.exec + type: test + params: + binary: bash + env: + VERSION_ID: ${version_id} + include_expansions_in_env: [PERF_URI_PRIVATE_ENDPOINT] + args: [*task-runner, perf-pr-comment] + run-enterprise-auth-tests: - command: ec2.assume_role params: @@ -684,6 +694,7 @@ tasks: binary: bash args: [*task-runner, driver-benchmark] - func: send-perf-data + - func: send-perf-pr-comment - name: test-standalone-noauth-nossl tags: ["test", "standalone"] diff --git a/Taskfile.yml b/Taskfile.yml index 2d6db5cf46..a4c6f405bf 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -70,6 +70,8 @@ tasks: pr-task: bash etc/pr-task.sh + perf-pr-comment: bash etc/perf-pr-comment.sh + # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect # specific operating systems or architectures. For example, staticcheck will only check for 64-bit # alignment of atomically accessed variables on 32-bit architectures (see diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh new file mode 100755 index 0000000000..ab7b9b8d23 --- /dev/null +++ b/etc/perf-pr-comment.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# perf-pr-comment +# Generates a report of Go Driver perf changes for the current branch. + +set -eux + +pushd ./internal/cmd/perfcomp >/dev/null || exist +GOWORK=off go run main.go --project="mongo-go-driver" ${VERSION_ID} +popd >/dev/null diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go new file mode 100644 index 0000000000..a41e24687d --- /dev/null +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -0,0 +1,98 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gonum.org/v1/gonum/mat" +) + +func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, step2 int) (*mat.Dense, *mat.Dense) { + xData := []float64{} + yData := []float64{} + + for i := start1; i < stop1; i += step1 { + xData = append(xData, float64(i)) + } + for j := start2; j < stop2; j += step2 { + yData = append(yData, float64(j)) + } + + x := mat.NewDense(len(xData), 1, xData) + y := mat.NewDense(len(yData), 1, yData) + + return x, y +} + +func TestEnergyStatistics(t *testing.T) { + t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { + x, y := createTestVectors(1, 100, 1, 1, 105, 1) + e, tstat, h, _ := getEnergyStatistics(x, y) + + del := 1e-3 + // Limit precision of comparison to 3 digits after the decimal. + assert.InDelta(t, 0.160, e, del) // |0.160 - e| < 0.001 + assert.InDelta(t, 8.136, tstat, del) + assert.InDelta(t, 0.002, h, del) + }) + + t.Run("different distributions should have large e,t,h values", func(t *testing.T) { + x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) + e, tstat, h, _ := getEnergyStatistics(x, y) + del := 1e-3 + + assert.InDelta(t, 21859.691, e, del) + assert.InDelta(t, 1481794.709, tstat, del) + assert.InDelta(t, 0.954, h, del) + }) + + t.Run("uni-variate distributions", func(t *testing.T) { + x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) + e, tstat, h, _ := getEnergyStatistics(x, y) + del := 1e-3 + + assert.InDelta(t, 4257.009, e, del) + assert.InDelta(t, 728381.015, tstat, del) + assert.InDelta(t, 0.748, h, del) + }) + + t.Run("equal distributions should have all 0 values", func(t *testing.T) { + x := mat.NewDense(10, 1, []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) + y := mat.NewDense(1, 1, []float64{1}) + + e, tstat, h, _ := getEnergyStatistics(x, y) + + assert.Equal(t, 0.0, e) + assert.Equal(t, 0.0, tstat) + assert.Equal(t, 0.0, h) + }) + + t.Run("energy stats returns errors on malformed input", func(t *testing.T) { + x := mat.NewDense(2, 2, make([]float64, 4)) + y := mat.NewDense(2, 3, make([]float64, 6)) + + _, _, _, err := getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "both inputs must have the same number of columns") + + x.Reset() + y = &mat.Dense{} + + _, _, _, err = getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "inputs cannot be empty") + + x = mat.NewDense(2, 2, make([]float64, 4)) + y = mat.NewDense(3, 2, make([]float64, 6)) + + _, _, _, err = getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "both inputs must be column vectors") + }) +} diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod new file mode 100644 index 0000000000..3b42147b1c --- /dev/null +++ b/internal/cmd/perfcomp/go.mod @@ -0,0 +1,28 @@ +module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp + +go 1.23.0 + +toolchain go1.23.10 + +replace go.mongodb.org/mongo-driver/v2 => ../../../ + +require ( + github.com/stretchr/testify v1.10.0 + go.mongodb.org/mongo-driver/v2 v2.2.2 + gonum.org/v1/gonum v0.16.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/text v0.23.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/internal/cmd/perfcomp/go.sum b/internal/cmd/perfcomp/go.sum new file mode 100644 index 0000000000..6d55f6e933 --- /dev/null +++ b/internal/cmd/perfcomp/go.sum @@ -0,0 +1,56 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go new file mode 100644 index 0000000000..e9c55d6d1f --- /dev/null +++ b/internal/cmd/perfcomp/main.go @@ -0,0 +1,428 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. +// Must use GOWORK=off to run this test. + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "math" + "os" + "strings" + "text/tabwriter" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" + "gonum.org/v1/gonum/mat" +) + +type OverrideInfo struct { + OverrideMainline bool `bson:"override_mainline"` + BaseOrder any `bson:"base_order"` + Reason any `bson:"reason"` + User any `bson:"user"` +} + +type Info struct { + Project string `bson:"project"` + Version string `bson:"version"` + Variant string `bson:"variant"` + Order int64 `bson:"order"` + TaskName string `bson:"task_name"` + TaskID string `bson:"task_id"` + Execution int64 `bson:"execution"` + Mainline bool `bson:"mainline"` + OverrideInfo OverrideInfo + TestName string `bson:"test_name"` + Args map[string]any `bson:"args"` +} + +type Stat struct { + Name string `bson:"name"` + Val float64 `bson:"val"` + Metadata any `bson:"metadata"` +} + +type Rollups struct { + Stats []Stat +} + +type RawData struct { + Info Info + CreatedAt any `bson:"created_at"` + CompletedAt any `bson:"completed_at"` + Rollups Rollups + FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` +} + +type TimeSeriesInfo struct { + Project string `bson:"project"` + Variant string `bson:"variant"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + Args map[string]any `bson:"args"` +} + +type StableRegion struct { + TimeSeriesInfo TimeSeriesInfo + Start any `bson:"start"` + End any `bson:"end"` + Values []float64 `bson:"values"` + StartOrder int64 `bson:"start_order"` + EndOrder int64 `bson:"end_order"` + Mean float64 `bson:"mean"` + Std float64 `bson:"std"` + Median float64 `bson:"median"` + Max float64 `bson:"max"` + Min float64 `bson:"min"` + CoefficientOfVariation float64 `bson:"coefficient_of_variation"` + LastSuccessfulUpdate any `bson:"last_successful_update"` + Last bool `bson:"last"` + Contexts []any `bson:"contexts"` +} + +type EnergyStats struct { + Project string + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + MeasurementVal float64 + PercentChange float64 + EnergyStatistic float64 + TestStatistic float64 + HScore float64 + ZScore float64 +} + +const expandedMetricsDB = "expanded_metrics" +const rawResultsColl = "raw_results" +const stableRegionsColl = "stable_regions" + +func main() { + // Check for variables + uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") + if uri == "" { + log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set") + } + + version := os.Args[len(os.Args)-1] + if version == "" { + log.Fatal("could not get VERSION_ID") + } + + // TODO (GODRIVER-3102): Map each project to a unique performance context, + // necessary for project switching to work since it's required for querying the stable region. + project := flag.String("project", "mongo-go-driver", "specify the name of an existing Evergreen project") + if project == nil { + log.Fatalf("must provide project") + } + + // Connect to analytics node + client, err := mongo.Connect(options.Client().ApplyURI(uri)) + if err != nil { + log.Fatalf("Error connecting client: %v", err) + } + + defer func() { // Defer disconnect client + err = client.Disconnect(context.Background()) + if err != nil { + log.Fatalf("Failed to disconnect client: %v", err) + } + }() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err = client.Ping(ctx, nil) + if err != nil { + log.Fatalf("Error pinging MongoDB Analytics: %v", err) + } + log.Println("Successfully connected to MongoDB Analytics node.") + + db := client.Database(expandedMetricsDB) + + // Get raw data, most recent stable region, and calculate energy stats + findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + patchRawData, err := findRawData(findCtx, *project, version, db.Collection(rawResultsColl)) + if err != nil { + log.Fatalf("Error getting raw data: %v", err) + } + + allEnergyStats, err := getEnergyStatsForAllBenchMarks(findCtx, patchRawData, db.Collection(stableRegionsColl)) + if err != nil { + log.Fatalf("Error getting energy statistics: %v", err) + } + log.Println(generatePRComment(allEnergyStats, version)) +} + +func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { + filter := bson.D{ + {"info.project", project}, + {"info.version", version}, + {"info.variant", "perf"}, + {"info.task_name", "perf"}, + } + + cursor, err := coll.Find(ctx, filter) + if err != nil { + log.Fatalf( + "Error retrieving raw data for version %q: %v", + version, + err, + ) + } + defer func() { + err = cursor.Close(ctx) + if err != nil { + log.Fatalf("Error closing cursor while retrieving raw data for version %q: %v", version, err) + } + }() + + log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) + + var rawData []RawData + err = cursor.All(ctx, &rawData) + if err != nil { + log.Fatalf( + "Error decoding raw data from version %q: %v", + version, + err, + ) + } + + return rawData, err +} + +// Find the most recent stable region of the mainline version for a specific test/measurement +func findLastStableRegion(ctx context.Context, project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { + filter := bson.D{ + {"time_series_info.project", project}, + {"time_series_info.variant", "perf"}, + {"time_series_info.task", "perf"}, + {"time_series_info.test", testname}, + {"time_series_info.measurement", measurement}, + {"last", true}, + {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, // TODO (GODRIVER-3102): Refactor perf context for project switching. + } + + findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) + + var sr *StableRegion + err := coll.FindOne(ctx, filter, findOptions).Decode(&sr) + if err != nil { + return nil, err + } + return sr, nil +} + +// For a specific test and measurement +func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + testname := rd.Info.TestName + var energyStats []*EnergyStats + + for i := range rd.Rollups.Stats { + project := rd.Info.Project + measName := rd.Rollups.Stats[i].Name + measVal := rd.Rollups.Stats[i].Val + + stableRegion, err := findLastStableRegion(ctx, project, testname, measName, coll) + if err != nil { + log.Fatalf( + "Error finding last stable region for test %q, measurement %q: %v", + testname, + measName, + err, + ) + } + + // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. + // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. + stableRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) + measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton + + estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) + if err != nil { + log.Fatalf( + "Could not calculate energy stats for test %q, measurement %q: %v", + testname, + measName, + err, + ) + } + + zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) + pChange := getPercentageChange(measVal, stableRegion.Mean) + + es := EnergyStats{ + Project: project, + Benchmark: testname, + Measurement: measName, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + MeasurementVal: measVal, + PercentChange: pChange, + EnergyStatistic: estat, + TestStatistic: tstat, + HScore: hscore, + ZScore: zscore, + } + energyStats = append(energyStats, &es) + } + + return energyStats, nil +} + +func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + var allEnergyStats []*EnergyStats + for _, rd := range patchRawData { + energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) + if err != nil { + log.Fatalf( + "Could not get energy stats for %q: %v", + rd.Info.TestName, + err, + ) + } else { + allEnergyStats = append(allEnergyStats, energyStats...) + } + } + return allEnergyStats, nil +} + +func generatePRComment(energyStats []*EnergyStats, version string) string { + var comment strings.Builder + comment.WriteString("# đź‘‹GoDriver Performance\n") + fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) + + w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) + fmt.Fprintln(w, "| Benchmark\t| Measurement\t| H-Score\t| Z-Score\t| % Change\t| Stable Reg\t| Patch Value\t|") + fmt.Fprintln(w, "| ---------\t| -----------\t| -------\t| -------\t| --------\t| ----------\t| -----------\t|") + + var testCount int64 + for _, es := range energyStats { + if math.Abs(es.ZScore) > 1.96 { + testCount += 1 + fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) + } + } + w.Flush() + + if testCount == 0 { + comment.Reset() + comment.WriteString("# đź‘‹GoDriver Performance\n") + comment.WriteString("There were no significant changes to the performance to report.") + } + + comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") + return comment.String() +} + +// Given two matrices, this function returns +// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) +func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != ycols { + return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") + } + if xrows == 0 || yrows == 0 { + return 0, 0, 0, fmt.Errorf("inputs cannot be empty") + } + + xrowsf := float64(xrows) + yrowsf := float64(yrows) + + var A float64 // E|X-Y| + if xrowsf > 0 && yrowsf > 0 { + dist, err := getDistance(x, y) + if err != nil { + return 0, 0, 0, err + } + A = dist / (xrowsf * yrowsf) + } else { + A = 0 + } + + var B float64 // E|X-X'| + if xrowsf > 0 { + dist, err := getDistance(x, x) + if err != nil { + return 0, 0, 0, err + } + B = dist / (xrowsf * xrowsf) + } else { + B = 0 + } + + var C float64 // E|Y-Y'| + if yrowsf > 0 { + dist, err := getDistance(y, y) + if err != nil { + return 0, 0, 0, err + } + C = dist / (yrowsf * yrowsf) + } else { + C = 0 + } + + E := 2*A - B - C // D^2(F_x, F_y) + T := ((xrowsf * yrowsf) / (xrowsf + yrowsf)) * E + var H float64 + if A > 0 { + H = E / (2 * A) + } else { + H = 0 + } + return E, T, H, nil +} + +// Given two vectors (expected 1 col), +// this function returns the sum of distances between each pair. +func getDistance(x, y *mat.Dense) (float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != 1 || ycols != 1 { + return 0, fmt.Errorf("both inputs must be column vectors") + } + + var sum float64 + + for i := 0; i < xrows; i++ { + for j := 0; j < yrows; j++ { + sum += math.Abs(x.At(i, 0) - y.At(j, 0)) + } + } + return sum, nil +} + +// Get Z score for result x, compared to mean u and st dev o. +func getZScore(x, mu, sigma float64) float64 { + if sigma == 0 { + return math.NaN() + } + return (x - mu) / sigma +} + +// Get percentage change for result x compared to mean u. +func getPercentageChange(x, mu float64) float64 { + if mu == 0 { + return math.NaN() + } + return ((x - mu) / mu) * 100 +}