store benchmark results in datastore

agent/lib/src/agent.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 import 'dart:convert';
+import 'dart:io';
 
 import 'package:args/args.dart';
 import 'package:http/http.dart';
@@ -68,10 +69,43 @@ class Agent {
     return null;
   }
 
-  Future<Null> updateTaskStatus(String taskKey, String newStatus) async {
+  Future<Null> reportSuccess(String taskKey, Map<String, dynamic> resultData, List<String> benchmarkScoreKeys) async {
+    var status = <String, dynamic>{
+      'TaskKey': taskKey,
+      'NewStatus': 'Succeeded',
+    };
+
+    // Make a copy of resultData because we may alter it during score key
+    // validation below.
+    resultData = resultData != null
+      ? new Map<String, dynamic>.from(resultData)
+      : <String, dynamic>{};
+    status['ResultData'] = resultData;
+
+    var validScoreKeys = <String>[];
+    if (benchmarkScoreKeys != null) {
+      for (String scoreKey in benchmarkScoreKeys) {
+        var score = resultData[scoreKey];
+        if (score is num) {
+          // Convert all metrics to double, which provide plenty of precision
+          // without having to add support for multiple numeric types on the
+          // backend.
+          resultData[scoreKey] = score.toDouble();
+          validScoreKeys.add(scoreKey);
+        } else {
+          stderr.writeln('WARNING: non-numeric score value $score submitted for $scoreKey');
+        }
+      }
+    }
+    status['BenchmarkScoreKeys'] = validScoreKeys;
+
+    await _cocoon('update-task-status', status);
+  }
+
+  Future<Null> reportFailure(String taskKey) async {
     await _cocoon('update-task-status', {
       'TaskKey': taskKey,
-      'NewStatus': newStatus,
+      'NewStatus': 'Failed',
     });
   }
 

agent/lib/src/commands/ci.dart

@@ -82,7 +82,7 @@ class ContinuousIntegrationCommand extends Command {
           }
         } catch(error, stackTrace) {
           print('ERROR: $error\n$stackTrace');
-          await agent.updateTaskStatus(task.key, 'Failed');
+          await agent.reportFailure(task.key);
         }
       } catch(error, stackTrace) {
         print('ERROR: $error\n$stackTrace');
@@ -99,10 +99,10 @@ class ContinuousIntegrationCommand extends Command {
     await runAndCaptureAsyncStacks(() async {
       TaskResult result = await runTask(agent, task);
       if (result.succeeded) {
-        await agent.updateTaskStatus(task.key, 'Succeeded');
+        await agent.reportSuccess(task.key, result.data, result.benchmarkScoreKeys);
         await _uploadDataToFirebase(task, result);
       } else {
-        await agent.updateTaskStatus(task.key, 'Failed');
+        await agent.reportFailure(task.key);
       }
     });
   }

app/index.yaml

@@ -30,3 +30,8 @@ indexes:
   - name: OwnerKey
   - name: CreateTimestamp
     direction: asc
+- kind: TimeseriesValue
+  ancestor: yes
+  properties:
+  - name: CreateTimestamp
+    direction: desc

commands/update_task_status.go

@@ -17,6 +17,10 @@ type UpdateTaskStatusCommand struct {
 	TaskKey *datastore.Key
 	// One of "Succeeded", "Failed".
 	NewStatus string
+	// If succeeded the result task data as JSON. nil otherwise.
+	ResultData map[string]interface{}
+	// Keys into the ResultData that represent a benchmark result
+	BenchmarkScoreKeys []string
 }
 
 // UpdateTaskStatusResult contains the updated task data.
@@ -44,6 +48,12 @@ func UpdateTaskStatus(c *db.Cocoon, inputJSON []byte) (interface{}, error) {
 		return nil, err
 	}
 
+	checklist, err := c.GetChecklist(task.Task.ChecklistKey)
+
+	if err != nil {
+		return nil, err
+	}
+
 	newStatus := db.TaskStatusByName(command.NewStatus)
 
 	if newStatus != db.TaskFailed {
@@ -62,5 +72,22 @@ func UpdateTaskStatus(c *db.Cocoon, inputJSON []byte) (interface{}, error) {
 
 	c.PutTask(task.Key, task.Task)
 
+	if newStatus == db.TaskSucceeded && len(command.BenchmarkScoreKeys) > 0 {
+		for _, scoreKey := range command.BenchmarkScoreKeys {
+			series, err := c.GetOrCreateTimeseries(scoreKey)
+
+			if err != nil {
+				return nil, err
+			}
+
+			value := command.ResultData[scoreKey].(float64)
+			_, err = c.SubmitTimeseriesValue(series, checklist.Checklist.Commit.Sha, task.Key, value)
+
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	return &UpdateTaskStatusResult{task}, nil
 }

db/db.go

@@ -583,6 +583,67 @@ func (t *Task) AgeInMillis() int64 {
 	return NowMillis() - t.CreateTimestamp
 }
 
+// GetOrCreateTimeseries fetches an existing timeseries, or creates and returns
+// a new one if one with the given scoreKey does not yet exist.
+func (c *Cocoon) GetOrCreateTimeseries(scoreKey string) (*TimeseriesEntity, error) {
+	key := datastore.NewKey(c.Ctx, "Timeseries", scoreKey, 0, nil)
+
+	var series *Timeseries
+	err := datastore.Get(c.Ctx, key, series)
+
+	if err == nil {
+		return &TimeseriesEntity{
+			Key:        key,
+			Timeseries: series,
+		}, nil
+	}
+
+	if err != datastore.ErrNoSuchEntity {
+		// Unexpected error, bail out.
+		return nil, err
+	}
+
+	// By default use scoreKey as label and "ms" as unit. It can be tweaked
+	// manually later using the datastore UI.
+	series = &Timeseries{
+		ID:    scoreKey,
+		Label: scoreKey,
+		Unit:  "ms",
+	}
+
+	_, err = datastore.Put(c.Ctx, key, series)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &TimeseriesEntity{
+		Key:        key,
+		Timeseries: series,
+	}, nil
+}
+
+// SubmitTimeseriesValue stores a TimeseriesValue in the datastore.
+func (c *Cocoon) SubmitTimeseriesValue(series *TimeseriesEntity, revision string,
+	taskKey *datastore.Key, value float64) (*TimeseriesValue, error) {
+	key := datastore.NewIncompleteKey(c.Ctx, "TimeseriesValue", series.Key)
+
+	timeseriesValue := &TimeseriesValue{
+		CreateTimestamp: NowMillis(),
+		Revision:        revision,
+		TaskKey:         taskKey,
+		Value:           value,
+	}
+
+	_, err := datastore.Put(c.Ctx, key, timeseriesValue)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return timeseriesValue, nil
+}
+
 // FetchURL performs an HTTP GET request on the given URL and returns data if
 // response is HTTP 200.
 func (c *Cocoon) FetchURL(url string) ([]byte, error) {

db/schema.go

@@ -81,6 +81,39 @@ type Task struct {
 	EndTimestamp       int64
 }
 
+// Timeseries contains a history of values of a certain performance metric.
+type Timeseries struct {
+	// Unique ID for computer consumption.
+	ID string
+	// A name used to display the series to humans.
+	Label string
+	// The unit used for the values, e.g. "ms", "kg", "pumpkins".
+	Unit string
+}
+
+// TimeseriesEntity contains storage data on a Timeseries.
+type TimeseriesEntity struct {
+	Key        *datastore.Key
+	Timeseries *Timeseries
+}
+
+// TimeseriesValue is a single value collected at a certain point in time at
+// a certain revision of Flutter.
+//
+// Entities of this type are stored as children of Timeseries and indexed by
+// CreateTimestamp in descencing order for faster access.
+type TimeseriesValue struct {
+	// The point in time this value was measured in milliseconds since the Unix
+	// epoch.
+	CreateTimestamp int64
+	// Flutter revision (git commit SHA)
+	Revision string
+	// The task that submitted the value.
+	TaskKey *datastore.Key
+	// The value.
+	Value float64
+}
+
 // MaxAttempts is the maximum number of times a single task will be attempted
 // before giving up on it.
 const MaxAttempts = 2