feat: add statistic tracking to agent-hub (#204)

* fix: unable to find agent with matching specs

* feat: add statistic tracking to agent-hub

packages/@best/agent-frontend/src/client/modules/view/dashboard/dashboard.css

@@ -12,4 +12,32 @@ header {
     background: rgb(255, 249, 199);
     border-radius: 4px;
     padding: 8px 0;
+}
+
+.stats {
+    background: #ffe8be;
+    padding: 2px 12px;
+    text-align: right;
+}
+
+.stats:first-of-type {
+    border-radius: 4px 4px 0 0;
+}
+
+.stats:last-of-type {
+    border-radius: 0 0 4px 4px;
+}
+
+.stats span {
+    font-size: 14px;
+    text-align: right;
+    display:inline;
+    padding-right:5px;
+}
+
+.stats .label {
+    font-weight: bold;
+}
+.stats .value {
+    font-size: 12px;
 }

packages/@best/agent-frontend/src/client/modules/view/dashboard/dashboard.html

@@ -1,5 +1,27 @@
 <template>
     <div class="container">
+        <template if:true={hasHubStats}>
+            <div class="stats">
+                <span class="label">Active Clients:</span>
+                <span class="value">{hubStats.runningClientCount}</span>
+                <span class="label">Pending Clients:</span>
+                <span class="value">{hubStats.pendingClientCount}</span>
+                <span class="label">Queued Jobs:</span>
+                <span class="value">{hubStats.pendingJobCount}</span>
+            </div>
+        </template>
+        <template if:true={hasAgentStats}>
+            <div class="stats">
+                <span class="label">Active Agents:</span>
+                <span class="value">{agentStats.active}</span>
+                <span class="label">Idle Agents:</span>
+                <span class="value">{agentStats.idle}</span>
+                <span class="label">Offline Agents:</span>
+                <span class="value">{agentStats.offline}</span>
+                <span class="label">Total:</span>
+                <span class="value">{agentStats.agents}</span>
+            </div>
+        </template>
         <template for:each={agents} for:item="agent">
             <component-agent key={agent.agentId} name={agent.agentId} jobs={agent.jobs}></component-agent>
         </template>

packages/@best/agent-frontend/src/client/modules/view/dashboard/dashboard.js

@@ -7,6 +7,8 @@ export default class ViewDashboard extends LightningElement {
     config = { host: window.location.origin, path: '/best' };
 
     @track agents = [];
+    @track hubStats = null;
+    @track agentStats = null;
     allJobs = [];
 
     connectedCallback() {
@@ -23,6 +25,7 @@ export default class ViewDashboard extends LightningElement {
         socket.on('benchmark error', this.error.bind(this));
         socket.on('benchmark cancel', this.cancel.bind(this));
         socket.on('benchmark results', this.results.bind(this));
+        socket.on('stats update', this.stats.bind(this));
     }
 
     // GETTERS
@@ -31,6 +34,14 @@ export default class ViewDashboard extends LightningElement {
         return this.allJobs.length > 0;
     }
 
+    get hasHubStats () {
+        return this.hubStats !== null;
+    }
+
+    get hasAgentStats () {
+        return this.agentStats !== null;
+    }
+
     // HELPERS
 
     addAgent(agentId) {
@@ -60,7 +71,7 @@ export default class ViewDashboard extends LightningElement {
         oldAgent.jobs.splice(oldJobIndex, 1);
 
         this.updateAgentsJob(updatedJob);
-    } 
+    }
 
     // SOCKET
 
@@ -104,7 +115,7 @@ export default class ViewDashboard extends LightningElement {
         const index = this.allJobs.findIndex(j => j.jobId === event.jobId);
         const job = this.allJobs[index];
         job.status = 'RUNNING';
-        
+
         if (event.agentId !== job.agentId) {
             const oldAgentId = job.agentId;
             job.agentId = event.agentId;
@@ -146,4 +157,9 @@ export default class ViewDashboard extends LightningElement {
 
         this.updateAgentsJob(job);
     }
+
+    stats(event) {
+        this.hubStats = event.packet.hub;
+        this.agentStats = event.packet.agentManager;
+    }
 }

packages/@best/agent-frontend/src/server/manager.ts

@@ -8,7 +8,7 @@
 import socketIO from 'socket.io';
 import AgentLogger from '@best/agent-logger';
 
-const FRONTEND_EVENTS = ['benchmark added', 'benchmark start', 'benchmark update', 'benchmark end', 'benchmark error', 'benchmark results', 'benchmark queued', 'benchmark cancel']
+const FRONTEND_EVENTS = ['benchmark added', 'benchmark start', 'benchmark update', 'benchmark end', 'benchmark error', 'benchmark results', 'benchmark queued', 'benchmark cancel', "stats update"]
 
 export default class Manager {
     private frontends: socketIO.Socket[] = [];

packages/@best/agent-hub/src/Agent.ts

@@ -62,10 +62,12 @@ export class Agent extends EventEmitter {
         if (value !== this._status) {
             const oldValue = this._status;
             this._status = value;
-            this.emit('status-changed', {
+            const packet = {
                 oldValue,
                 newValue: value
-            })
+            }
+            this.emit('status-changed', packet);
+            this._logger.event("AgentManager", "AGENT_STATUS_CHANGED", packet);
         }
     }
 

packages/@best/agent-hub/src/AgentManager.ts

@@ -10,6 +10,16 @@ import {Agent, AgentConfig, AgentStatus, Spec} from "./Agent";
 import BenchmarkJob from "./BenchmarkJob";
 import AgentLogger from "@best/agent-logger";
 
+// Wait time 30 minutes
+const REMOVE_OFFLINE_AGENT_WAIT = 1800000;
+
+export interface AgentManagerStatus {
+    agents: number
+    active: number,
+    idle: number,
+    offline: number
+}
+
 export class AgentManager extends EventEmitter {
     private agents: Agent[] = [];
 
@@ -45,21 +55,65 @@ export class AgentManager extends EventEmitter {
         this.addStatusChangeListener(agent);
     }
 
+    removeAgent(agent: Agent) {
+        // remove agent and emit event.
+        this.emit("agent-removed", this.agents.splice(this.agents.indexOf(agent), 1));
+    }
+
+    removeAgentByHostname(agentHost: string) {
+        const agent = this.getAgent(agentHost);
+        if (agent !== null) {
+            this.removeAgent(agent);
+        }
+    }
+
     private addStatusChangeListener = (agent: Agent) => {
         agent.on('status-changed', ({ newValue }: { newValue: AgentStatus }) => {
             if (newValue === AgentStatus.Idle) {
                 this.emit('idle-agent', agent);
+            } else if (newValue === AgentStatus.Offline) {
+                setTimeout(() => {
+                    // remove the agent after a specific amount of time if it remains offline.
+                    if (agent.status === AgentStatus.Offline) {
+                        this.removeAgent(agent);
+                    }
+                }, REMOVE_OFFLINE_AGENT_WAIT)
             }
         });
     };
 
-    getAgent(agentHost: string) {
+    getAgent(agentHost: string): Agent | null {
         let i: number = 0;
 
         while (i < this.agents.length && this.agents[i].host !== agentHost) i++;
 
         return i < this.agents.length ? this.agents[i] : null;
     }
+
+    getAgentsStatus() : AgentManagerStatus {
+        const ret: AgentManagerStatus = {
+            agents: this.agents.length,
+            active: 0,
+            idle: 0,
+            offline: 0,
+        }
+
+        this.agents.forEach(agent => {
+            switch(agent.status) {
+                case AgentStatus.Idle:
+                    ret.idle++;
+                    break;
+                case AgentStatus.Offline:
+                    ret.offline++;
+                    break;
+                case AgentStatus.RunningJob:
+                    ret.active++;
+                    break;
+            }
+        })
+
+        return ret;
+    }
 }
 
 export function createAgentManager(agentsConfig: AgentConfig[], logger: AgentLogger): AgentManager {

packages/@best/agent-hub/src/HubApplication.ts

@@ -13,6 +13,12 @@ import {Agent, Spec} from "./Agent";
 import {Client} from "./Client";
 import AgentLogger from "@best/agent-logger";
 
+export interface HubStatus {
+    pendingJobCount: number
+    pendingClientCount: number,
+    runningClientCount: number,
+}
+
 export class HubApplication {
     private _incomingQueue: ObservableQueue<BenchmarkJob>;
     private _agentManager: AgentManager;
@@ -31,6 +37,15 @@ export class HubApplication {
         this.attachEventListeners();
     }
 
+    getLoadStatus() : HubStatus {
+        return {
+            pendingJobCount: this.pendingClients.reduce((count, client) => count + client.jobsToRun, 0)
+            + this.runningClients.reduce((count, client) => count + client.jobsToRun, 0) + this._incomingQueue.size,
+            pendingClientCount: this.pendingClients.length,
+            runningClientCount: this.runningClients.length
+        }
+    }
+
     handleIncomingSocketConnection = (socket: SocketIO.Socket) => {
         // @todo: define the types for the data.
         // @todo: add timeout on waiting the benchmark task?
@@ -81,6 +96,9 @@ export class HubApplication {
                             }
                         }
                     }
+
+                    this._logger.event("Hub", 'CLIENT_DISCONNECTED');
+
                 });
 
                 const notAssignedAgents = this.agentManager.getAgentsForSpec(spec)
@@ -93,6 +111,8 @@ export class HubApplication {
                     this.pendingClients.push(client);
                     client.notifyInQueue(this.pendingClients.length);
                 }
+
+                this._logger.event("Hub", 'CLIENT_CONNECTED');
             }
         });
     };
@@ -104,6 +124,7 @@ export class HubApplication {
     private attachEventListeners() {
         // @todo: rename the event to match each other
         this._agentManager.on('idle-agent', this.handleIdleAgent);
+        this._agentManager.on("agent-removed", this.handleRemovedAgent)
         this._incomingQueue.on('item-added', this.handleJobReadyToRun);
     }
 
@@ -113,18 +134,31 @@ export class HubApplication {
         if (agent !== null) {
             this._incomingQueue.remove(job);
             agent.runJob(job);
+            this._logger.event("Hub", 'PENDING_JOB_CHANGED');
         } else {
             job.socketConnection.emit('benchmark_enqueued', { pending: this._incomingQueue.size });
             this._logger.event(job.socketConnection.id, 'benchmark queued', { pending: this._incomingQueue.size }, false);
         }
     };
 
+    private handleRemovedAgent = (agent: Agent) => {
+        if (this.assignedAgents.has(agent)) {
+            const client: Client | undefined = this.assignedAgents.get(agent);
+            this.assignedAgents.delete(agent);
+            if (client) {
+                this.clientAgents.delete(client);
+            }
+            this._logger.event("Hub", 'AGENT_STATUS_CHANGED');
+        }
+    }
+
     private handleIdleAgent = (agent: Agent) => {
         if (this.assignedAgents.has(agent)) {
             const assignedClient = this.assignedAgents.get(agent)!;
 
             if (assignedClient.jobsToRun > 0) {
                 assignedClient.askForJob();
+                this._logger.event("Hub", 'PENDING_JOB_CHANGED');
             } else {
                 this.askJobFromRunningClientWithAgentSpecs(agent);
             }
@@ -154,6 +188,7 @@ export class HubApplication {
         for (let i=0, n = Math.min(notAssignAndIdleAgents.length, client.jobsToRun); i< n; i++) {
             client.askForJob();
         }
+        this._logger.event("Hub", 'PENDING_JOB_CHANGED');
     }
 
     private handleIdleFreeAgent(agent: Agent) {
@@ -174,6 +209,7 @@ export class HubApplication {
 
         if (runningClientsForAgent.length > 0) {
             runningClientsForAgent[0].askForJob();
+            this._logger.event("Hub", 'PENDING_JOB_CHANGED');
         }
     }
 
@@ -182,6 +218,7 @@ export class HubApplication {
         for (const job of this._incomingQueue) {
             if (agent.canRunJob(job!.spec)) {
                 this._incomingQueue.remove(job!);
+                this._logger.event("Hub", 'PENDING_JOB_CHANGED');
                 agent.runJob(job!);
                 break;
             }