AUC chart

libs/core-ui/src/index.ts

@@ -53,6 +53,7 @@ export * from "./lib/util/getCommonStyles";
 export * from "./lib/util/getCompositeFilterString";
 export * from "./lib/util/getFeatureOptions";
 export * from "./lib/util/getFilterBoundsArgs";
+export * from "./lib/util/calculateAUCData";
 export * from "./lib/util/calculateBoxData";
 export * from "./lib/util/calculateConfusionMatrixData";
 export * from "./lib/util/calculateLineData";

libs/core-ui/src/lib/FeatureFlights.ts

@@ -2,10 +2,14 @@
 // Licensed under the MIT License.
 
 export const dataBalanceExperienceFlight = "dataBalanceExperience";
+export const aucChartExperienceFlight = "aucChartExperience";
 export const featureFlightSeparator = "&";
 
 // add more entries for new feature flights
-export const featureFlights = [dataBalanceExperienceFlight];
+export const featureFlights = [
+  dataBalanceExperienceFlight,
+  aucChartExperienceFlight
+];
 
 export function parseFeatureFlights(featureFlights?: string): string[] {
   if (featureFlights) {

libs/core-ui/src/lib/Interfaces/IAUCData.ts

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export interface IAUCData {
+  AUCData: number[][];
+  selectedLabels: string[];
+}

libs/core-ui/src/lib/components/OverallMetricChartUtils.ts

@@ -220,6 +220,7 @@ export function generateMetricsList(
   if (!chartProps) {
     return [];
   }
+  console.log(yAxisProperty);
   if (chartProps.yAxis.property === cohortKey) {
     const indexes = context.errorCohorts.map((errorCohort) =>
       errorCohort.cohort.unwrap(JointDataset.IndexLabel)

libs/core-ui/src/lib/util/StatisticsUtils.ts

@@ -245,7 +245,6 @@ const generateImageStats: (
     }
   ];
 };
-
 export const generateMetrics: (
   jointDataset: JointDataset,
   selectionIndexes: number[][],

libs/core-ui/src/lib/util/calculateAUCData.test.ts

@@ -0,0 +1,148 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { binarizeData, calculatePerClassROCData } from "./calculateAUCData";
+
+describe("Test binarizeData", () => {
+  it("should binarize numbers", () => {
+    const result = binarizeData([1, 3, 4, 0], [0, 1, 2, 3, 4]);
+    expect(result).toEqual([
+      [0, 1, 0, 0, 0],
+      [0, 0, 0, 1, 0],
+      [0, 0, 0, 0, 1],
+      [1, 0, 0, 0, 0]
+    ]);
+  });
+  it("should binarize strings", () => {
+    const result = binarizeData(
+      ["one", "two", "three"],
+      ["three", "one", "two"]
+    );
+    expect(result).toEqual([
+      [0, 1, 0],
+      [0, 0, 1],
+      [1, 0, 0]
+    ]);
+  });
+  it("should binarize binary data", () => {
+    const result = binarizeData([1, 0, 1, 0], [0, 1]);
+    expect(result).toEqual([
+      [0, 1],
+      [1, 0],
+      [0, 1],
+      [1, 0]
+    ]);
+  });
+});
+describe("Test calculatePerClassROCData", () => {
+  it("generate x,y data corresponding to fpr and tpr respectively", () => {
+    const result = calculatePerClassROCData(
+      [0.33, 0.32, 0.34, 0.29, 0.12, 0.41, 0.4, 0.39],
+      [0, 1, 1, 0, 1, 0, 0, 0]
+    );
+    expect(result).toEqual({
+      points: [
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 1 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 1, y: 0.6666666666666666 },
+        { x: 0.8, y: 0.6666666666666666 },
+        { x: 0.8, y: 0.6666666666666666 },
+        { x: 0.8, y: 0.6666666666666666 },
+        { x: 0.8, y: 0.3333333333333333 },
+        { x: 0.6, y: 0.3333333333333333 },
+        { x: 0.6, y: 0 },
+        { x: 0.6, y: 0 },
+        { x: 0.6, y: 0 },
+        { x: 0.6, y: 0 },
+        { x: 0.6, y: 0 },
+        { x: 0.4, y: 0 },
+        { x: 0.2, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 },
+        { x: 0, y: 0 }
+      ]
+    });
+  });
+});

libs/core-ui/src/lib/util/calculateAUCData.ts

@@ -0,0 +1,142 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { localization } from "@responsible-ai/localization";
+import { SeriesOptionsType } from "highcharts";
+import { range, unzip } from "lodash";
+
+import { IDataset } from "../Interfaces/IDataset";
+
+interface IPoint {
+  x: number;
+  y: number;
+}
+export interface IROCData {
+  points: IPoint[];
+}
+
+function getStaticROCData(): SeriesOptionsType[] {
+  return [
+    {
+      data: [
+        { x: 0, y: 0 },
+        { x: 0, y: 1 },
+        { x: 1, y: 1 }
+      ],
+      name: localization.Interpret.Charts.Ideal,
+      type: "line"
+    },
+    {
+      data: [
+        { x: 0, y: 0 },
+        { x: 1, y: 1 }
+      ],
+      name: localization.Interpret.Charts.Random,
+      type: "line"
+    }
+  ];
+}
+
+export function calculatePerClassROCData(
+  probabilityY: number[],
+  binY: number[]
+): IROCData {
+  const rocData: IROCData = {
+    points: []
+  };
+  const thresholds = range(0, 1, 0.01);
+  let truePositives = 0;
+  let falsePositives = 0;
+  let trueNegatives = 0;
+  let falseNegatives = 0;
+
+  for (const threshold of thresholds) {
+    for (const [index, yProba] of probabilityY.entries()) {
+      // if the probability of predicting the positive label is greater than the
+      // threshold then it's a true positive.
+      // otherwise, it's a false positive
+      if (yProba < threshold) {
+        if (binY[index]) {
+          falseNegatives++;
+        } else {
+          trueNegatives++;
+        }
+      } else if (binY[index]) {
+        truePositives++;
+      } else {
+        falsePositives++;
+      }
+    }
+    addROCPoint(
+      truePositives,
+      falsePositives,
+      trueNegatives,
+      falseNegatives,
+      rocData
+    );
+    truePositives = falsePositives = trueNegatives = falseNegatives = 0;
+  }
+
+  return rocData;
+}
+
+function addROCPoint(
+  truePositives: number,
+  falsePositives: number,
+  trueNegatives: number,
+  falseNegatives: number,
+  rocData: IROCData
+): void {
+  // prevent division by 0
+  const totalNegatives = trueNegatives + falsePositives;
+  const totalPositives = truePositives + falseNegatives;
+  const tpr = totalPositives === 0 ? 1 : truePositives / totalPositives;
+  const fpr = totalNegatives === 0 ? 1 : falsePositives / totalNegatives;
+  rocData.points.push({ x: fpr, y: tpr });
+}
+
+export function binarizeData(
+  yData: string[] | number[] | number[][],
+  classes: string[] | number[]
+): number[][] {
+  // binarize labels in a one-vs-all fashion according to
+  const yBinData: number[][] = [];
+  for (const yDatum of yData) {
+    const binaryData = classes.map((c) => {
+      return c === yDatum ? 1 : 0;
+    });
+    yBinData.push(binaryData);
+  }
+  return yBinData;
+}
+
+// based on https://msdata.visualstudio.com/Vienna/_git/AzureMlCli?path=/src/azureml-metrics/azureml/metrics/_classification.py&version=GBmaster
+export function calculateAUCData(dataset: IDataset): SeriesOptionsType[] {
+  if (!dataset.probability_y || !dataset.class_names) {
+    // TODO: show warning message
+    return [...getStaticROCData()];
+  }
+
+  // temporary, replace with dataset.classnames
+  const cNames = [0, 1];
+  const binTrueY = binarizeData(dataset.true_y, cNames);
+  console.log(binTrueY);
+  // transpose in order to group class data together
+  const perClassBinY = unzip(binTrueY);
+  const perClassProba = unzip(dataset.probability_y);
+  const data = [];
+  // loop through each class to calculate roc data per class
+  for (const [i, element] of perClassBinY.entries()) {
+    const classROCData = calculatePerClassROCData(perClassProba[i], element);
+    const classData = {
+      data: classROCData.points,
+      // TODO: check class_names length earlier ?
+      name: cNames ? cNames[i] : "",
+      type: "line"
+    };
+    data.push(classData);
+  }
+
+  const allData = [...data, ...getStaticROCData()];
+  return allData as SeriesOptionsType[];
+}

libs/localization/src/lib/en.json

@@ -838,7 +838,9 @@
       "rowIndex": "Row index",
       "absoluteIndex": "Absolute index",
       "xValue": "X-value",
-      "yValue": "Y-value"
+      "yValue": "Y-value",
+      "Ideal": "Ideal",
+      "Random": "Random"
     },
     "Cohort": {
       "_cohort.comment": "a subset of the data is called a cohort",
@@ -1818,6 +1820,7 @@
       "regressionDistributionPivotItem": "Target distribution",
       "metricsVisualizationsPivotItem": "Metrics visualizations",
       "confusionMatrixPivotItem": "Confusion matrix",
+      "AUCPivotItem": "AUC Chart",
       "disaggregatedAnalysisFeatureSelectionPlaceholder": "Select features to generate the feature-based analysis.",
       "tableCountTooltip": "Cohort {0} contains {1} instances.",
       "tableMetricTooltip": "The model's {0} on cohort {1} is {2}",