report & show agent health; color-code flaky successes

agent/bin/agent.dart

@@ -59,7 +59,7 @@ Future<Null> main(List<String> rawArgs) async {
     exit(1);
   }
 
-  print('Agent configuration:');
+  section('Agent configuration:');
   print(config);
 
   await command.run(args.command);

agent/lib/src/adb.dart

@@ -46,6 +46,22 @@ class Adb {
   // 015d172c98400a03       device usb:340787200X product:nakasi model:Nexus_7 device:grouper
   static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)');
 
+  static Future<Map<String, HealthCheckResult>> checkDevices() async {
+    Map<String, HealthCheckResult> results = <String, HealthCheckResult>{};
+    for (String deviceId in await deviceIds) {
+      try {
+        Adb device = new Adb(deviceId: deviceId);
+        // Just a smoke test that we can read wakefulness state
+        // TODO(yjbanov): check battery level
+        await device._getWakefulness();
+        results['android-device-$deviceId'] = new HealthCheckResult.success();
+      } catch(e, s) {
+        results['android-device-$deviceId'] = new HealthCheckResult.error(e, s);
+      }
+    }
+    return results;
+  }
+
   static Future<List<String>> get deviceIds async {
     List<String> output = (await eval(config.adbPath, ['devices', '-l'], canFail: false))
         .trim().split('\n');

agent/lib/src/agent.dart

@@ -127,8 +127,16 @@ class Agent {
     });
   }
 
-  Future<Map<String, dynamic>> getAuthenticationStatus() async {
-    return await _cocoon('get-authentication-status');
+  Future<String> getAuthenticationStatus() async {
+    return (await _cocoon('get-authentication-status'))['Status'];
+  }
+
+  Future<Null> updateHealthStatus(AgentHealth health) async {
+    await _cocoon('update-agent-health', {
+    	'AgentID': agentId,
+    	'IsHealthy': health.ok,
+    	'HealthDetails': '$health',
+    });
   }
 }
 
@@ -162,3 +170,36 @@ abstract class Command {
 
   Future<Null> run(ArgResults args);
 }
+
+/// Overall health of the agent.
+class AgentHealth {
+  /// Check results keyed by parameter.
+  final Map<String, HealthCheckResult> checks = <String, HealthCheckResult>{};
+
+  /// Whether all [checks] succeeded.
+  bool get ok => checks.isNotEmpty && checks.values.every((HealthCheckResult r) => r.succeeded);
+
+  /// Sets a health check [result] for a given [parameter].
+  operator []=(String parameter, HealthCheckResult result) {
+    if (checks.containsKey(parameter)) {
+      print('WARNING: duplicate health check ${parameter} submitted');
+    }
+    checks[parameter] = result;
+  }
+
+  void addAll(Map<String, HealthCheckResult> checks) {
+    checks.forEach((String p, HealthCheckResult r) {
+      this[p] = r;
+    });
+  }
+
+  /// Human-readable printout of the agent's health status.
+  @override
+  String toString() {
+    StringBuffer buf = new StringBuffer();
+    checks.forEach((String parameter, HealthCheckResult result) {
+      buf.writeln('$parameter: $result');
+    });
+    return buf.toString();
+  }
+}

agent/lib/src/commands/ci.dart

@@ -31,10 +31,36 @@ class ContinuousIntegrationCommand extends Command {
 
   @override
   Future<Null> run(ArgResults args) async {
-    await _performPreflightChecks();
+    // Perform one pre-flight round of checks and quit immediately if something
+    // is wrong.
+    AgentHealth health = await _performHealthChecks();
+    section('Pre-flight checks:');
+    print(health);
+
+    if (!health.ok) {
+      print('Some pre-flight checks failed. Quitting.');
+      exit(1);
+    }
+
+    // Start CI mode
+    section('Started continuous integration:');
     _listenToShutdownSignals();
     while(!_exiting) {
       try {
+        // Check health before requesting a new task.
+        health = await _performHealthChecks();
+
+        // Always upload health status whether succeeded or failed.
+        await agent.updateHealthStatus(health);
+
+        if (!health.ok) {
+          print('Some health checks failed:');
+          print(health);
+          await new Future.delayed(_sleepBetweenBuilds);
+          // Don't bother requesting new tasks if health is bad.
+          continue;
+        }
+
         CocoonTask task = await agent.reserveTask();
         try {
           if (task != null) {
@@ -67,22 +93,45 @@ class ContinuousIntegrationCommand extends Command {
         await forceQuitRunningProcesses();
       }
 
-      // TODO(yjbanov): report health status after running the task
       await new Future.delayed(_sleepBetweenBuilds);
     }
   }
 
-  Future<Null> _performPreflightChecks() async {
-    print('Pre-flight checks:');
-    await pickNextDevice();
-    print('  - device connected');
-    await checkFirebaseConnection();
-    print('  - firebase connected');
-    if (!(await agent.getAuthenticationStatus())['Status'] == 'OK') {
-      throw 'Failed to authenticate to Cocoon. Check config.yaml.';
+  Future<AgentHealth> _performHealthChecks() async {
+    AgentHealth results = new AgentHealth();
+    try {
+      results['firebase-connection'] = await checkFirebaseConnection();
+
+      Map<String, HealthCheckResult> deviceChecks = await Adb.checkDevices();
+      results.addAll(deviceChecks);
+
+      int healthyDeviceCount = deviceChecks.values
+        .where((HealthCheckResult r) => r.succeeded)
+        .length;
+
+      results['has-healthy-devices'] = healthyDeviceCount > 0
+        ? new HealthCheckResult.success('Found ${deviceChecks.length} healthy devices')
+        : new HealthCheckResult.failure('No healthy devices found');
+
+      try {
+        String authStatus = await agent.getAuthenticationStatus();
+        results['cocoon-connection'] = new HealthCheckResult.success();
+
+        if (authStatus != 'OK') {
+          results['cocoon-authentication'] = new HealthCheckResult.failure('Failed to authenticate to Cocoon. Check config.yaml.');
+        } else {
+          results['cocoon-authentication'] = new HealthCheckResult.success();
+        }
+      } catch(e, s) {
+        results['cocoon-connection'] = new HealthCheckResult.error(e, s);
+      }
+
+      results['able-to-perform-health-check'] = new HealthCheckResult.success();
+    } catch(e, s) {
+      results['able-to-perform-health-check'] = new HealthCheckResult.error(e, s);
     }
-    print('  - Cocoon auth OK');
-    print('Pre-flight OK');
+
+    return results;
   }
 
   /// Listens to standard output and upload logs to Cocoon in semi-realtime.

agent/lib/src/firebase.dart

@@ -16,9 +16,17 @@ Firebase _measurements() {
       auth: firebaseToken);
 }
 
-Future<Null> checkFirebaseConnection() async {
-  if (await _measurements().child('dashboard_bot_status').child('current').get() == null) {
-    throw 'Connection to Firebase is unhealthy. Failed to read the current dashboard_bot_status entity.';
+Future<HealthCheckResult> checkFirebaseConnection() async {
+  try {
+    if ((await _measurements().child('dashboard_bot_status').child('current').get()).val == null) {
+      return new HealthCheckResult.failure(
+        'Connection to Firebase is unhealthy. Failed to read the current dashboard_bot_status entity.'
+      );
+    } else {
+      return new HealthCheckResult.success();
+    }
+  } catch (e, s) {
+    return new HealthCheckResult.error(e, s);
   }
 }
 

agent/lib/src/utils.dart

@@ -38,6 +38,31 @@ class ProcessInfo {
   }
 }
 
+/// Result of a health check for a specific parameter.
+class HealthCheckResult {
+  HealthCheckResult.success([this.details]) : succeeded = true;
+  HealthCheckResult.failure(this.details) : succeeded = false;
+  HealthCheckResult.error(dynamic error, dynamic stackTrace)
+    : succeeded = false,
+      details = 'ERROR: $error\n${stackTrace ?? ''}';
+
+  final bool succeeded;
+  final String details;
+
+  @override
+  String toString() {
+    StringBuffer buf = new StringBuffer(succeeded ? 'succeeded' : 'failed');
+    if (details != null && details.trim().isNotEmpty) {
+      buf.writeln();
+      // Indent details by 4 spaces
+      for (String line in details.trim().split('\n')) {
+        buf.writeln('    $line');
+      }
+    }
+    return '$buf';
+  }
+}
+
 class BuildFailedError extends Error {
   BuildFailedError(this.message);
 

app/lib/components/status_table.dart

@@ -16,6 +16,34 @@ import 'package:http/http.dart' as http;
 <div *ngIf="isLoading" style="position: fixed; top: 0; left: 0; background-color: #AAFFAA;">
   Loading...
 </div>
+
+<div class="agent-bar">
+  <div>Agents</div>
+  <div *ngFor="let agentStatus of agentStatuses"
+       [ngClass]="getAgentStyle(agentStatus)"
+       (click)="showAgentHealthDetails(agentStatus)">
+    {{agentStatus.agentId}}
+  </div>
+</div>
+
+<div *ngIf="displayedAgentStatus != null"
+     class="agent-health-details-card">
+  <div style="position: absolute; top: 5px; right: 5px; cursor: pointer"
+       (click)="hideAgentHealthDetails()">
+    [X]
+  </div>
+  <div>
+    {{displayedAgentStatus.agentId}}
+    {{isAgentHealthy(displayedAgentStatus) ? "☺" : "☹"}}
+  </div>
+  <div>
+    Last health check: {{displayedAgentStatus.healthCheckTimestamp}}
+    {{agentHealthCheckAge(displayedAgentStatus.healthCheckTimestamp)}}
+  </div>
+  <div>Details:</div>
+  <div>{{displayedAgentStatus.healthDetails}}</div>
+</div>
+
 <table class="status-table"
        cellspacing="0"
        cellpadding="0"
@@ -47,6 +75,8 @@ import 'package:http/http.dart' as http;
   directives: const [NgIf, NgFor, NgClass]
 )
 class StatusTable implements OnInit {
+  static const Duration maxHealthCheckAge = const Duration(minutes: 10);
+
   StatusTable(this._httpClient);
 
   final http.Client _httpClient;
@@ -61,6 +91,8 @@ class StatusTable implements OnInit {
   /// A sparse Commit X Task matrix of test results.
   Map<String, Map<String, TaskEntity>> resultMatrix = <String, Map<String, TaskEntity>>{};
 
+  List<AgentStatus> agentStatuses;
+
   @override
   ngOnInit() async {
     reloadData();
@@ -73,6 +105,7 @@ class StatusTable implements OnInit {
     GetStatusResult statusResult = GetStatusResult.fromJson(statusJson);
     isLoading = false;
 
+    agentStatuses = statusResult.agentStatuses ?? <AgentStatus>[];
     List<BuildStatus> statuses = statusResult.statuses ?? <BuildStatus>[];
     headerCol = <BuildStatus>[];
     headerRow = new HeaderRow();
@@ -95,7 +128,7 @@ class StatusTable implements OnInit {
     return fullSha.length > 7 ? fullSha.substring(0, 7) : fullSha;
   }
 
-  List<String> taskStatusToCssStyle(String taskStatus) {
+  List<String> taskStatusToCssStyle(String taskStatus, int attempts) {
     const statusMap = const <String, String> {
       'New': 'task-new',
       'In Progress': 'task-in-progress',
@@ -104,7 +137,15 @@ class StatusTable implements OnInit {
       'Underperformed': 'task-underperformed',
       'Skipped': 'task-skipped',
     };
-    return ['task-status-circle', statusMap[taskStatus] ?? 'task-unknown'];
+
+    String cssClass;
+    if (taskStatus == 'Succeeded' && attempts > 1) {
+      cssClass = 'task-succeeded-but-flaky';
+    } else {
+      cssClass = statusMap[taskStatus] ?? 'task-unknown';
+    }
+
+    return ['task-status-circle', cssClass];
   }
 
   TaskEntity _findTask(String sha, String taskName) {
@@ -120,9 +161,43 @@ class StatusTable implements OnInit {
     TaskEntity taskEntity = _findTask(sha, taskName);
 
     if (taskEntity == null)
-      return taskStatusToCssStyle('Skipped');
+      return taskStatusToCssStyle('Skipped', 0);
+
+    return taskStatusToCssStyle(taskEntity.task.status, taskEntity.task.attempts);
+  }
+
+  List<String> getAgentStyle(AgentStatus status) {
+    return [
+      'agent-chip',
+      isAgentHealthy(status) ? 'agent-healthy' : 'agent-unhealthy',
+    ];
+  }
+
+  /// An agent is considered healthy if the latest health report was OK and is
+  /// up-to-date.
+  bool isAgentHealthy(AgentStatus status) {
+    return status.isHealthy && status.healthCheckTimestamp != null &&
+      new DateTime.now().difference(status.healthCheckTimestamp) < maxHealthCheckAge;
+  }
+
+  AgentStatus displayedAgentStatus;
+
+  void showAgentHealthDetails(AgentStatus agentStatus) {
+    displayedAgentStatus = agentStatus;
+  }
+
+  void hideAgentHealthDetails() {
+    displayedAgentStatus = null;
+  }
 
-    return taskStatusToCssStyle(taskEntity.task.status);
+  String agentHealthCheckAge(DateTime dt) {
+    if (dt == null)
+      return '';
+    Duration age = new DateTime.now().difference(dt);
+    String ageQualifier = age > maxHealthCheckAge
+      ? 'out-of-date!!!'
+      : 'old';
+    return '(${age.inMinutes} minutes $ageQualifier)';
   }
 
   void openLog(String sha, String taskName) {

app/lib/entity.dart

@@ -30,6 +30,9 @@ abstract class JsonSerializer<T> {
 /// Serializes strings.
 StringSerializer string() => const StringSerializer();
 
+/// Serializes booleans.
+BoolSerializer boolean() => const BoolSerializer();
+
 /// Serializes ints and doubles.
 NumSerializer number() => const NumSerializer();
 
@@ -83,6 +86,15 @@ class StringSerializer implements JsonSerializer<String> {
   dynamic serialize(String value) => value;
 }
 
+class BoolSerializer implements JsonSerializer<bool> {
+  const BoolSerializer();
+
+  bool deserialize(dynamic jsonValue) {
+    return jsonValue as bool;
+  }
+  dynamic serialize(bool value) => value;
+}
+
 class NumSerializer implements JsonSerializer<num> {
   const NumSerializer();