diff --git a/agent/lib/src/agent.dart b/agent/lib/src/agent.dart index 67fb4f25bf..bed53cff39 100644 --- a/agent/lib/src/agent.dart +++ b/agent/lib/src/agent.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'package:args/args.dart'; import 'package:http/http.dart'; @@ -68,10 +69,43 @@ class Agent { return null; } - Future updateTaskStatus(String taskKey, String newStatus) async { + Future reportSuccess(String taskKey, Map resultData, List benchmarkScoreKeys) async { + var status = { + 'TaskKey': taskKey, + 'NewStatus': 'Succeeded', + }; + + // Make a copy of resultData because we may alter it during score key + // validation below. + resultData = resultData != null + ? new Map.from(resultData) + : {}; + status['ResultData'] = resultData; + + var validScoreKeys = []; + if (benchmarkScoreKeys != null) { + for (String scoreKey in benchmarkScoreKeys) { + var score = resultData[scoreKey]; + if (score is num) { + // Convert all metrics to double, which provide plenty of precision + // without having to add support for multiple numeric types on the + // backend. + resultData[scoreKey] = score.toDouble(); + validScoreKeys.add(scoreKey); + } else { + stderr.writeln('WARNING: non-numeric score value $score submitted for $scoreKey'); + } + } + } + status['BenchmarkScoreKeys'] = validScoreKeys; + + await _cocoon('update-task-status', status); + } + + Future reportFailure(String taskKey) async { await _cocoon('update-task-status', { 'TaskKey': taskKey, - 'NewStatus': newStatus, + 'NewStatus': 'Failed', }); } diff --git a/agent/lib/src/commands/ci.dart b/agent/lib/src/commands/ci.dart index 27652ee84e..e25c793e45 100644 --- a/agent/lib/src/commands/ci.dart +++ b/agent/lib/src/commands/ci.dart @@ -82,7 +82,7 @@ class ContinuousIntegrationCommand extends Command { } } catch(error, stackTrace) { print('ERROR: $error\n$stackTrace'); - await agent.updateTaskStatus(task.key, 'Failed'); + await agent.reportFailure(task.key); } } catch(error, stackTrace) { print('ERROR: $error\n$stackTrace'); @@ -99,10 +99,10 @@ class ContinuousIntegrationCommand extends Command { await runAndCaptureAsyncStacks(() async { TaskResult result = await runTask(agent, task); if (result.succeeded) { - await agent.updateTaskStatus(task.key, 'Succeeded'); + await agent.reportSuccess(task.key, result.data, result.benchmarkScoreKeys); await _uploadDataToFirebase(task, result); } else { - await agent.updateTaskStatus(task.key, 'Failed'); + await agent.reportFailure(task.key); } }); } diff --git a/app/index.yaml b/app/index.yaml index d146a676d9..6fe1292a15 100644 --- a/app/index.yaml +++ b/app/index.yaml @@ -30,3 +30,8 @@ indexes: - name: OwnerKey - name: CreateTimestamp direction: asc +- kind: TimeseriesValue + ancestor: yes + properties: + - name: CreateTimestamp + direction: desc diff --git a/commands/update_task_status.go b/commands/update_task_status.go index 00b0d9237e..cf55a90146 100644 --- a/commands/update_task_status.go +++ b/commands/update_task_status.go @@ -17,6 +17,10 @@ type UpdateTaskStatusCommand struct { TaskKey *datastore.Key // One of "Succeeded", "Failed". NewStatus string + // If succeeded the result task data as JSON. nil otherwise. + ResultData map[string]interface{} + // Keys into the ResultData that represent a benchmark result + BenchmarkScoreKeys []string } // UpdateTaskStatusResult contains the updated task data. @@ -44,6 +48,12 @@ func UpdateTaskStatus(c *db.Cocoon, inputJSON []byte) (interface{}, error) { return nil, err } + checklist, err := c.GetChecklist(task.Task.ChecklistKey) + + if err != nil { + return nil, err + } + newStatus := db.TaskStatusByName(command.NewStatus) if newStatus != db.TaskFailed { @@ -62,5 +72,22 @@ func UpdateTaskStatus(c *db.Cocoon, inputJSON []byte) (interface{}, error) { c.PutTask(task.Key, task.Task) + if newStatus == db.TaskSucceeded && len(command.BenchmarkScoreKeys) > 0 { + for _, scoreKey := range command.BenchmarkScoreKeys { + series, err := c.GetOrCreateTimeseries(scoreKey) + + if err != nil { + return nil, err + } + + value := command.ResultData[scoreKey].(float64) + _, err = c.SubmitTimeseriesValue(series, checklist.Checklist.Commit.Sha, task.Key, value) + + if err != nil { + return nil, err + } + } + } + return &UpdateTaskStatusResult{task}, nil } diff --git a/db/db.go b/db/db.go index f5e906f844..8152a4c5af 100644 --- a/db/db.go +++ b/db/db.go @@ -583,6 +583,67 @@ func (t *Task) AgeInMillis() int64 { return NowMillis() - t.CreateTimestamp } +// GetOrCreateTimeseries fetches an existing timeseries, or creates and returns +// a new one if one with the given scoreKey does not yet exist. +func (c *Cocoon) GetOrCreateTimeseries(scoreKey string) (*TimeseriesEntity, error) { + key := datastore.NewKey(c.Ctx, "Timeseries", scoreKey, 0, nil) + + var series *Timeseries + err := datastore.Get(c.Ctx, key, series) + + if err == nil { + return &TimeseriesEntity{ + Key: key, + Timeseries: series, + }, nil + } + + if err != datastore.ErrNoSuchEntity { + // Unexpected error, bail out. + return nil, err + } + + // By default use scoreKey as label and "ms" as unit. It can be tweaked + // manually later using the datastore UI. + series = &Timeseries{ + ID: scoreKey, + Label: scoreKey, + Unit: "ms", + } + + _, err = datastore.Put(c.Ctx, key, series) + + if err != nil { + return nil, err + } + + return &TimeseriesEntity{ + Key: key, + Timeseries: series, + }, nil +} + +// SubmitTimeseriesValue stores a TimeseriesValue in the datastore. +func (c *Cocoon) SubmitTimeseriesValue(series *TimeseriesEntity, revision string, + taskKey *datastore.Key, value float64) (*TimeseriesValue, error) { + key := datastore.NewIncompleteKey(c.Ctx, "TimeseriesValue", series.Key) + + timeseriesValue := &TimeseriesValue{ + CreateTimestamp: NowMillis(), + Revision: revision, + TaskKey: taskKey, + Value: value, + } + + _, err := datastore.Put(c.Ctx, key, timeseriesValue) + + if err != nil { + return nil, err + } + + return timeseriesValue, nil +} + // FetchURL performs an HTTP GET request on the given URL and returns data if // response is HTTP 200. func (c *Cocoon) FetchURL(url string) ([]byte, error) { diff --git a/db/schema.go b/db/schema.go index 4fb38c2724..18087d149d 100644 --- a/db/schema.go +++ b/db/schema.go @@ -81,6 +81,39 @@ type Task struct { EndTimestamp int64 } +// Timeseries contains a history of values of a certain performance metric. +type Timeseries struct { + // Unique ID for computer consumption. + ID string + // A name used to display the series to humans. + Label string + // The unit used for the values, e.g. "ms", "kg", "pumpkins". + Unit string +} + +// TimeseriesEntity contains storage data on a Timeseries. +type TimeseriesEntity struct { + Key *datastore.Key + Timeseries *Timeseries +} + +// TimeseriesValue is a single value collected at a certain point in time at +// a certain revision of Flutter. +// +// Entities of this type are stored as children of Timeseries and indexed by +// CreateTimestamp in descencing order for faster access. +type TimeseriesValue struct { + // The point in time this value was measured in milliseconds since the Unix + // epoch. + CreateTimestamp int64 + // Flutter revision (git commit SHA) + Revision string + // The task that submitted the value. + TaskKey *datastore.Key + // The value. + Value float64 +} + // MaxAttempts is the maximum number of times a single task will be attempted // before giving up on it. const MaxAttempts = 2