[feat] initial impl of apptesting (#8799)

* Basic init flow

* run formatter

* execute command

* wip changes to call new api

* more fully implement the flow with actual API interactions

* add tests and run formatter

* a little clean up

* carry displayname through

* small lint stuff

* teak the operation response handling

* Refactor to remove the intermediate TestDef type and parse directly to TestCaseInvocation

* make sure apptesting is disabled by default

* fix typo in console url

* use legacy webId for console URL + some final polish

* add a schema file

* resolving some review comments

* remove browser from yaml spec

* polish and fix some of the polling output

* update CHANGELOG

* fix tests

* revert mocha import changes

* fix a broken conditional

* fix typo and log invocation id

* run format

* format

---------

Co-authored-by: kjelko <kjelko@google.com>

Author: kjelko

CHANGELOG.md

@@ -0,0 +1 @@
+- Add experimental App Testing feature

schema/apptesting-yaml.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "additionalProperties": false,
+  "definitions": {
+    "testConfig": {
+      "additionalProperties": false,
+      "type": "object",
+      "properties": {
+        "route": {
+          "type": "string",
+          "description": "URL route appended to the test target base URL at which to start the test"
+        }
+      }
+    },
+    "testStep": {
+      "additionalProperties": false,
+      "properties": {
+        "goal": {
+          "type": "string",
+          "description": "The goal of the test step"
+        },
+        "hint": {
+          "type": "string",
+          "description": "A hint to provide extra context to accomplish the goal"
+        },
+        "successCriteria": {
+          "type": "string",
+          "description": "Crtieria that the agent can use determine if the goal is complete"
+        }
+      }
+    },
+    "test": {
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "A descriptive name of the test"
+        },
+        "testConfig": {
+          "$ref": "#/definitions/testConfig",
+          "description": "Configs to apply to the specific test"
+        },
+        "steps": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/testStep"
+          }
+        }
+      }
+    }
+  },
+  "properties": {
+    "defaultConfig": {
+      "$ref": "#/definitions/testConfig",
+      "description": "Default config to apply to each test within the file"
+    },
+    "tests": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/test"
+      }
+    }
+  }
+}

src/api.ts

@@ -170,6 +170,9 @@ export const vertexAIOrigin = () =>
 export const cloudAiCompanionOrigin = () =>
   utils.envOverride("CLOUD_AI_COMPANION_URL", "https://cloudaicompanion.googleapis.com");
 
+export const appTestingOrigin = () =>
+  utils.envOverride("FIREBASE_APP_TESTING_URL", "https://firebaseapptesting.googleapis.com");
+
 /** Gets scopes that have been set. */
 export function getScopes(): string[] {
   return Array.from(commandScopes);

src/apptesting/invokeTests.spec.ts

@@ -0,0 +1,152 @@
+import { expect } from "chai";
+import * as nock from "nock";
+import { appTestingOrigin } from "../api";
+import { invokeTests, pollInvocationStatus } from "./invokeTests";
+import { FirebaseError } from "../error";
+import { Browser } from "./types";
+
+describe("invokeTests", () => {
+  describe("invokeTests", () => {
+    const projectNumber = "123456789";
+    const appId = `1:${projectNumber}:ios:abc123def456`;
+
+    it("throws FirebaseError if invocation request fails", async () => {
+      nock(appTestingOrigin())
+        .post(`/v1alpha/projects/${projectNumber}/apps/${appId}/testInvocations:invokeTestCases`)
+        .reply(400, { error: {} });
+      await expect(invokeTests(appId, "https://www.example.com", [])).to.be.rejectedWith(
+        FirebaseError,
+        "Test invocation failed",
+      );
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("returns operation when successful", async () => {
+      nock(appTestingOrigin())
+        .post(`/v1alpha/projects/${projectNumber}/apps/${appId}/testInvocations:invokeTestCases`)
+        .reply(200, { name: "foo/bar/biz" });
+      const operation = await invokeTests(appId, "https://www.example.com", []);
+      expect(operation).to.eql({ name: "foo/bar/biz" });
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("builds the correct request", async () => {
+      let requestBody;
+      nock(appTestingOrigin())
+        .post(
+          `/v1alpha/projects/${projectNumber}/apps/${appId}/testInvocations:invokeTestCases`,
+          (r) => {
+            requestBody = r;
+            return true;
+          },
+        )
+        .reply(200, { name: "foo/bar/biz" });
+
+      await invokeTests(appId, "https://www.example.com", [
+        {
+          testCase: {
+            startUri: "https://www.example.com",
+            displayName: "testName1",
+            instructions: { steps: [{ goal: "test this app", hint: "try clicking the button" }] },
+          },
+          testExecution: [{ config: { browser: Browser.CHROME } }],
+        },
+        {
+          testCase: {
+            startUri: "https://www.example.com",
+            displayName: "testName2",
+            instructions: { steps: [{ goal: "retest it", successCriteria: "a dialog appears" }] },
+          },
+          testExecution: [{ config: { browser: Browser.CHROME } }],
+        },
+      ]);
+
+      expect(requestBody).to.eql({
+        resource: {
+          testCaseInvocations: [
+            {
+              testCase: {
+                displayName: "testName1",
+                instructions: {
+                  steps: [
+                    {
+                      goal: "test this app",
+                      hint: "try clicking the button",
+                    },
+                  ],
+                },
+                startUri: "https://www.example.com",
+              },
+              testExecution: [
+                {
+                  config: {
+                    browser: "CHROME",
+                  },
+                },
+              ],
+            },
+            {
+              testCase: {
+                displayName: "testName2",
+                instructions: {
+                  steps: [
+                    {
+                      goal: "retest it",
+                      successCriteria: "a dialog appears",
+                    },
+                  ],
+                },
+                startUri: "https://www.example.com",
+              },
+              testExecution: [
+                {
+                  config: {
+                    browser: "CHROME",
+                  },
+                },
+              ],
+            },
+          ],
+          testInvocation: {},
+        },
+      });
+    });
+  });
+
+  describe("pollInvocationStatus", () => {
+    const operationName = "operations/foo/bar";
+
+    beforeEach(() => {
+      nock(appTestingOrigin())
+        .get(`/v1alpha/${operationName}`)
+        .reply(200, { done: false, metadata: { count: 1 } });
+      nock(appTestingOrigin())
+        .get(`/v1alpha/${operationName}`)
+        .reply(200, { done: false, metadata: { count: 2 } });
+      nock(appTestingOrigin())
+        .get(`/v1alpha/${operationName}`)
+        .reply(200, { done: true, metadata: { count: 3 }, response: { foo: "12" } });
+    });
+
+    it("calls poll callback with metadata on each poll", async () => {
+      const pollResponses: { [k: string]: any }[] = [];
+      await pollInvocationStatus(
+        operationName,
+        (op) => {
+          pollResponses.push(op.metadata!);
+        },
+        /* backoff= */ 1,
+      );
+
+      expect(pollResponses).to.eql([{ count: 1 }, { count: 2 }, { count: 3 }]);
+      expect(nock.isDone()).to.be.true;
+    });
+
+    it("returns the response", async () => {
+      const response = await pollInvocationStatus(operationName, () => null, /* backoff= */ 1);
+
+      expect(response).to.eql({ foo: "12" });
+      expect(nock.isDone()).to.be.true;
+    });
+  });
+});

src/apptesting/invokeTests.ts

@@ -0,0 +1,57 @@
+import { Client } from "../apiv2";
+import { appTestingOrigin } from "../api";
+import {
+  InvokedTestCases,
+  InvokeTestCasesRequest,
+  TestCaseInvocation,
+  TestInvocation,
+} from "./types";
+import * as operationPoller from "../operation-poller";
+import { FirebaseError, getError } from "../error";
+
+const apiClient = new Client({ urlPrefix: appTestingOrigin(), apiVersion: "v1alpha" });
+
+export async function invokeTests(appId: string, startUri: string, testDefs: TestCaseInvocation[]) {
+  const appResource = `projects/${appId.split(":")[1]}/apps/${appId}`;
+  try {
+    const invocationResponse = await apiClient.post<
+      InvokeTestCasesRequest,
+      operationPoller.LongRunningOperation<TestInvocation>
+    >(`${appResource}/testInvocations:invokeTestCases`, buildInvokeTestCasesRequest(testDefs));
+    return invocationResponse.body;
+  } catch (err: unknown) {
+    throw new FirebaseError("Test invocation failed", { original: getError(err) });
+  }
+}
+
+function buildInvokeTestCasesRequest(
+  testCaseInvocations: TestCaseInvocation[],
+): InvokeTestCasesRequest {
+  return {
+    resource: {
+      testInvocation: {},
+      testCaseInvocations,
+    },
+  };
+}
+
+interface InvocationOperation {
+  resource: InvokedTestCases;
+}
+
+export async function pollInvocationStatus(
+  operationName: string,
+  onPoll: (invocation: operationPoller.OperationResult<InvocationOperation>) => void,
+  backoff = 30 * 1000,
+): Promise<InvocationOperation> {
+  return operationPoller.pollOperation<InvocationOperation>({
+    pollerName: "App Testing Invocation Poller",
+    apiOrigin: appTestingOrigin(),
+    apiVersion: "v1alpha",
+    operationResourceName: operationName,
+    masterTimeout: 30 * 60 * 1000, // 30 minutes
+    backoff,
+    maxBackoff: 15 * 1000, // 30 seconds
+    onPoll,
+  });
+}