Removed Aggregate Method for Model Overview Metrics with Object Detection

apps/dashboard/src/model-assessment-vision/App.tsx

@@ -51,7 +51,7 @@ export class App extends React.Component<IAppProps> {
       ...this.props,
       locale: this.props.language,
       localUrl: "https://www.bing.com/",
-      requestObjectDetectionMetrics: (selectionIndexes, _, className) =>
+      requestObjectDetectionMetrics: (selectionIndexes, className) =>
         this.generateRandomObjectDetectionMetrics(selectionIndexes, className),
       stringParams: { contextualHelp: this.messages },
       theme: this.props.theme

apps/widget/src/app/ModelAssessment.tsx

@@ -27,7 +27,6 @@ export class ModelAssessment extends React.Component<IModelAssessmentProps> {
       };
       callBack.requestObjectDetectionMetrics = async (
         selectionIndexes: number[][],
-        aggregateMethod: string,
         className: string,
         iouThreshold: number,
         objectDetectionCache: Map<string, [number, number, number]>,
@@ -37,7 +36,6 @@ export class ModelAssessment extends React.Component<IModelAssessmentProps> {
           this.props.config,
           [
             selectionIndexes,
-            aggregateMethod,
             className,
             iouThreshold,
             objectDetectionCache

libs/core-ui/src/lib/Context/ModelAssessmentContext.tsx

@@ -143,7 +143,6 @@ export interface IModelAssessmentContext {
   requestObjectDetectionMetrics?:
     | ((
         selectionIndexes: number[][],
-        aggregateMethod: string,
         className: string,
         iouThreshold: number,
         objectDetectionCache: Map<string, [number, number, number]>,

libs/core-ui/src/lib/util/ObjectDetectionStatisticsUtils.ts

@@ -17,20 +17,19 @@ export enum ObjectDetectionMetrics {
 export const generateObjectDetectionStats: (
   selectionIndexes: number[][],
   objectDetectionCache: Map<string, [number, number, number]>,
-  objectDetectionInputs: [string, string, number]
+  objectDetectionInputs: [string, number]
 ) => ILabeledStatistic[][] = (
   selectionIndexes: number[][],
   objectDetectionCache: Map<string, [number, number, number]>,
-  objectDetectionInputs: [string, string, number]
+  objectDetectionInputs: [string, number]
 ): ILabeledStatistic[][] => {
   return selectionIndexes.map((selectionArray) => {
     const count = selectionArray.length;
 
-    const key: [number[], string, string, number] = [
+    const key: [number[], string, number] = [
       selectionArray,
       objectDetectionInputs[0],
       objectDetectionInputs[1],
-      objectDetectionInputs[2]
     ];
     const value = objectDetectionCache.get(key.toString());
     const stat = value ? value : [Number.NaN, Number.NaN, Number.NaN];

libs/core-ui/src/lib/util/StatisticsUtils.ts

@@ -148,14 +148,14 @@ export const generateMetrics: (
   selectionIndexes: number[][],
   modelType: ModelTypes,
   objectDetectionCache?: Map<string, [number, number, number]>,
-  objectDetectionInputs?: [string, string, number],
+  objectDetectionInputs?: [string, number],
   questionAnsweringCache?: QuestionAnsweringCacheType
 ) => ILabeledStatistic[][] = (
   jointDataset: JointDataset,
   selectionIndexes: number[][],
   modelType: ModelTypes,
   objectDetectionCache?: Map<string, [number, number, number]>,
-  objectDetectionInputs?: [string, string, number],
+  objectDetectionInputs?: [string, number],
   questionAnsweringCache?: QuestionAnsweringCacheType
 ): ILabeledStatistic[][] => {
   if (

libs/interpret-vision/src/lib/VisionExplanationDashboard/Interfaces/IVisionExplanationDashboardProps.ts

@@ -18,7 +18,6 @@ export interface IVisionExplanationDashboardProps {
   ) => Promise<any[]>;
   requestObjectDetectionMetrics?: (
     selectionIndexes: number[][],
-    aggregateMethod: string,
     className: string,
     iouThreshold: number,
     objectDetectionCache: Map<string, [number, number, number]>,

libs/model-assessment/src/lib/ModelAssessmentDashboard/Controls/ModelOverview/ModelOverview.tsx

@@ -55,7 +55,6 @@ interface IModelOverviewProps {
   telemetryHook?: (message: ITelemetryEvent) => void;
   requestObjectDetectionMetrics?: (
     selectionIndexes: number[][],
-    aggregateMethod: string,
     className: string,
     iouThreshold: number,
     objectDetectionCache: Map<string, [number, number, number]>
@@ -77,7 +76,6 @@ interface IModelOverviewState {
   selectedFeatureBasedCohorts?: number[];
   chartConfigurationIsVisible: boolean;
   datasetCohortViewIsVisible: boolean;
-  aggregateMethod: string;
   datasetCohortChartIsVisible: boolean;
   featureConfigurationIsVisible: boolean;
   metricConfigurationIsVisible: boolean;
@@ -112,8 +110,6 @@ export class ModelOverview extends React.Component<
     super(props);
 
     this.state = {
-      aggregateMethod:
-        localization.ModelAssessment.ModelOverview.metricTypes.macro,
       chartConfigurationIsVisible: false,
       className: "",
       datasetBasedCohorts: [],
@@ -357,7 +353,6 @@ export class ModelOverview extends React.Component<
               <ObjectDetectionWidgets
                 classNames={classNames}
                 dataset={this.context.dataset}
-                setAggregateMethod={this.setAggregateMethod}
                 setClassName={this.setClassName}
                 setIoUThreshold={this.setIoUThreshold}
                 updateDatasetCohortStats={this.updateDatasetCohortStats}
@@ -566,20 +561,6 @@ export class ModelOverview extends React.Component<
     );
   }
 
-  private setAggregateMethod = (value: string): void => {
-    this.setState({ aggregateMethod: value }, () => {
-      if (this.state.datasetCohortChartIsVisible) {
-        this.updateDatasetCohortStats();
-      } else {
-        this.updateFeatureCohortStats();
-      }
-    });
-
-    this.logButtonClick(
-      TelemetryEventName.ModelOverviewMetricsSelectionUpdated
-    );
-  };
-
   private setClassName = (value: string): void => {
     this.setState({ className: value }, () => {
       if (this.state.datasetCohortChartIsVisible) {
@@ -618,7 +599,6 @@ export class ModelOverview extends React.Component<
       this.context.modelMetadata.modelType,
       this.objectDetectionCache,
       [
-        this.state.aggregateMethod,
         this.state.className,
         this.state.iouThreshold
       ],
@@ -646,14 +626,12 @@ export class ModelOverview extends React.Component<
     if (
       this.context.requestObjectDetectionMetrics &&
       selectionIndexes.length > 0 &&
-      this.state.aggregateMethod.length > 0 &&
       this.state.className.length > 0 &&
       this.state.iouThreshold
     ) {
       this.context
         .requestObjectDetectionMetrics(
           selectionIndexes,
-          this.state.aggregateMethod,
           this.state.className,
           this.state.iouThreshold,
           this.objectDetectionCache,
@@ -683,9 +661,8 @@ export class ModelOverview extends React.Component<
               for (const [i, cohortMetric] of cohortMetrics.entries()) {
                 const [mAP, aP, aR] = cohortMetric;
 
-                const key: [number[], string, string, number] = [
+                const key: [number[], string, number] = [
                   selectionIndexes[cohortIndex],
-                  this.state.aggregateMethod,
                   cohortClasses[i],
                   this.state.iouThreshold
                 ];
@@ -861,7 +838,6 @@ export class ModelOverview extends React.Component<
       this.context.modelMetadata.modelType,
       this.objectDetectionCache,
       [
-        this.state.aggregateMethod,
         this.state.className,
         this.state.iouThreshold
       ],

libs/model-assessment/src/lib/ModelAssessmentDashboard/Controls/ModelOverview/ObjectDetectionModelOverview.tsx

@@ -20,19 +20,6 @@ import React from "react";
 
 import { IModelOverviewStyles } from "./ModelOverview.styles";
 
-export function getSelectableAggregateMethod(): IComboBoxOption[] {
-  const selectableAggregateMethods: IComboBoxOption[] = [
-    {
-      key: "macro",
-      text: localization.ModelAssessment.ModelOverview.metricTypes.macro
-    },
-    {
-      key: "micro",
-      text: localization.ModelAssessment.ModelOverview.metricTypes.micro
-    }
-  ];
-  return selectableAggregateMethods;
-}
 
 export function getSelectableClassNames(dataset: IDataset): IComboBoxOption[] {
   const selectableClassNames: IComboBoxOption[] = [];
@@ -50,7 +37,6 @@ export function getSelectableClassNames(dataset: IDataset): IComboBoxOption[] {
 export interface IObjectDetectionWidgetsProps {
   classNames: IProcessedStyleSet<IModelOverviewStyles>;
   dataset: IDataset;
-  setAggregateMethod: (value: string) => void;
   setClassName: (value: string) => void;
   setIoUThreshold: (value: number) => void;
   updateDatasetCohortStats: () => void;
@@ -62,15 +48,6 @@ export class ObjectDetectionWidgets extends React.PureComponent<IObjectDetection
   public render(): React.ReactNode {
     return (
       <Stack horizontal tokens={{ childrenGap: "10px" }}>
-        <ComboBox
-          id="modelOverviewAggregateMethod"
-          label={localization.ModelAssessment.ModelOverview.metricsTypeDropdown}
-          defaultSelectedKey={"macro"}
-          options={getSelectableAggregateMethod()}
-          onChange={this.onAggregateMethodChange}
-          className={this.props.classNames.dropdown}
-          styles={FluentUIStyles.smallDropdownStyle}
-        />
         <ComboBox
           id="modelOverviewClassSelection"
           placeholder={
@@ -115,15 +92,6 @@ export class ObjectDetectionWidgets extends React.PureComponent<IObjectDetection
     );
   }
 
-  private onAggregateMethodChange = (
-    _: React.FormEvent<IComboBox>,
-    item?: IComboBoxOption
-  ): void => {
-    if (item) {
-      this.props.setAggregateMethod(item.text.toString());
-    }
-  };
-
   private onClassNameChange = (
     _: React.FormEvent<IComboBox>,
     item?: IComboBoxOption

libs/model-assessment/src/lib/ModelAssessmentDashboard/Controls/TabsView/TabsViewProps.ts

@@ -46,7 +46,6 @@ export interface ITabsViewProps {
   ) => Promise<any[]>;
   requestObjectDetectionMetrics?: (
     selectionIndexes: number[][],
-    aggregateMethod: string,
     className: string,
     iouThreshold: number,
     objectDetectionCache: Map<string, [number, number, number]>,

libs/model-assessment/src/lib/ModelAssessmentDashboard/ModelAssessmentDashboardProps.ts

@@ -117,7 +117,6 @@ export interface IModelAssessmentDashboardProps
   ) => Promise<any[]>;
   requestObjectDetectionMetrics?: (
     selectionIndexes: number[][],
-    aggregateMethod: string,
     className: string,
     iouThreshold: number,
     objectDetectionCache: Map<string, [number, number, number]>,

raiwidgets/raiwidgets/responsibleai_dashboard_input.py

@@ -331,13 +331,11 @@ def get_object_detection_metrics(self, post_data):
         """
         try:
             selection_indexes = post_data[0]
-            aggregate_method = post_data[1]
-            class_name = post_data[2]
-            iou_threshold = post_data[3]
-            object_detection_cache = post_data[4]
+            class_name = post_data[1]
+            iou_threshold = post_data[2]
+            object_detection_cache = post_data[3]
             exp = self._analysis.compute_object_detection_metrics(
                 selection_indexes,
-                aggregate_method,
                 class_name,
                 iou_threshold,
                 object_detection_cache

responsibleai_vision/requirements.txt

@@ -6,5 +6,6 @@ scikit-learn>=0.22.1
 scipy>=1.4.1
 semver~=2.13.0
 responsibleai>=0.29.0
+pycocotools
 torchmetrics
 vision_explanation_methods

responsibleai_vision/responsibleai_vision/rai_vision_insights/rai_vision_insights.py

@@ -1084,30 +1084,25 @@ def load(path):
     def compute_object_detection_metrics(
             self,
             selection_indexes,
-            aggregate_method,
             class_name,
             iou_threshold,
             object_detection_cache):
         dashboard_dataset = self.get_data().dataset
         true_y = dashboard_dataset.object_detection_true_y
         predicted_y = dashboard_dataset.object_detection_predicted_y
-        dashboard_dataset = self.get_data().dataset
-        true_y = dashboard_dataset.object_detection_true_y
-        predicted_y = dashboard_dataset.object_detection_predicted_y
 
         normalized_iou_threshold = [iou_threshold / 100.0]
         all_cohort_metrics = []
         for cohort_indices in selection_indexes:
             key = ','.join([str(cid) for cid in cohort_indices] +
-                           [aggregate_method, class_name, str(iou_threshold)])
+                           [class_name, str(iou_threshold)])
             if key in object_detection_cache:
                 all_cohort_metrics.append(object_detection_cache[key])
                 continue
 
             metric_OD = MeanAveragePrecision(
                 class_metrics=True,
-                iou_thresholds=normalized_iou_threshold,
-                average=aggregate_method)
+                iou_thresholds=normalized_iou_threshold)
             true_y_cohort = [true_y[cohort_index] for cohort_index
                              in cohort_indices]
             predicted_y_cohort = [predicted_y[cohort_index] for cohort_index