From 11b53b8b8e1c79f84040347fcc4d9e7ff013bce6 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Mon, 6 Oct 2025 12:10:03 -0400 Subject: [PATCH 01/19] creation of feature branch From 18c29f88700c269bfc9270e8c0435ebda1ebc304 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Mon, 6 Oct 2025 15:09:55 -0400 Subject: [PATCH 02/19] New MCP tool for running mobile tests (via app distribution). (#9250) * Scaffolding for new appdistribution MCP tool. * Refactor business logic out of the appdistribution CLI so that it can be used by an MCP tool. * Wire new appdistribution tool up to the business logic. * Fix linting errors. * Update src/appdistribution/distribution.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --------- Co-authored-by: Jamie Rothfeder Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/appdistribution/client.ts | 2 +- src/appdistribution/distribution.ts | 230 ++++++++++++++++++++- src/appdistribution/options-parser-util.ts | 9 +- src/commands/appdistribution-distribute.ts | 228 ++------------------ src/mcp/prompts/index.ts | 1 + src/mcp/tools/appdistribution/index.ts | 4 + src/mcp/tools/appdistribution/tests.ts | 40 ++++ src/mcp/tools/index.ts | 2 + src/mcp/types.ts | 1 + src/mcp/util.ts | 2 + 10 files changed, 299 insertions(+), 220 deletions(-) create mode 100644 src/mcp/tools/appdistribution/index.ts create mode 100644 src/mcp/tools/appdistribution/tests.ts diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 07af4b456c8..839e3d8bdcc 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -66,7 +66,7 @@ export class AppDistributionClient { }); } - async updateReleaseNotes(releaseName: string, releaseNotes: string): Promise { + async updateReleaseNotes(releaseName: string, releaseNotes?: string): Promise { if (!releaseNotes) { utils.logWarning("no release notes specified, skipping"); return; diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index 3f03d7ea35b..30c46620965 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -1,7 +1,17 @@ import * as fs from "fs-extra"; -import { FirebaseError, getErrMsg } from "../error"; import { logger } from "../logger"; import * as pathUtil from "path"; +import * as utils from "../utils"; +import { + AabInfo, + IntegrationState, + UploadReleaseResult, + TestDevice, + ReleaseTest, + LoginCredential, +} from "../appdistribution/types"; +import { AppDistributionClient } from "./client"; +import { FirebaseError, getErrMsg, getErrStatus } from "../error"; export enum DistributionFileType { IPA = "ipa", @@ -9,6 +19,168 @@ export enum DistributionFileType { AAB = "aab", } +const TEST_MAX_POLLING_RETRIES = 40; +const TEST_POLLING_INTERVAL_MILLIS = 30_000; + +/** + * Execute an app distribution action + */ +export async function distribute( + appName: string, + distribution: Distribution, + testCases: string[], + testDevices: TestDevice[], + releaseNotes?: string, + testers?: string[], + groups?: string[], + testNonBlocking?: boolean, + loginCredential?: LoginCredential, +) { + const requests = new AppDistributionClient(); + let aabInfo: AabInfo | undefined; + if (distribution.distributionFileType() === DistributionFileType.AAB) { + try { + aabInfo = await requests.getAabInfo(appName); + } catch (err: unknown) { + if (getErrStatus(err) === 404) { + throw new FirebaseError( + `App Distribution could not find your app ${appName}. ` + + `Make sure to onboard your app by pressing the "Get started" ` + + "button on the App Distribution page in the Firebase console: " + + "https://console.firebase.google.com/project/_/appdistribution", + { exit: 1 }, + ); + } + throw new FirebaseError(`failed to determine AAB info. ${getErrMsg(err)}`, { exit: 1 }); + } + + if ( + aabInfo.integrationState !== IntegrationState.INTEGRATED && + aabInfo.integrationState !== IntegrationState.AAB_STATE_UNAVAILABLE + ) { + switch (aabInfo.integrationState) { + case IntegrationState.PLAY_ACCOUNT_NOT_LINKED: { + throw new FirebaseError("This project is not linked to a Google Play account."); + } + case IntegrationState.APP_NOT_PUBLISHED: { + throw new FirebaseError('"This app is not published in the Google Play console.'); + } + case IntegrationState.NO_APP_WITH_GIVEN_BUNDLE_ID_IN_PLAY_ACCOUNT: { + throw new FirebaseError("App with matching package name does not exist in Google Play."); + } + case IntegrationState.PLAY_IAS_TERMS_NOT_ACCEPTED: { + throw new FirebaseError( + "You must accept the Play Internal App Sharing (IAS) terms to upload AABs.", + ); + } + default: { + throw new FirebaseError( + "App Distribution failed to process the AAB: " + aabInfo.integrationState, + ); + } + } + } + } + + utils.logBullet("uploading binary..."); + let releaseName; + try { + const operationName = await requests.uploadRelease(appName, distribution); + + // The upload process is asynchronous, so poll to figure out when the upload has finished successfully + const uploadResponse = await requests.pollUploadStatus(operationName); + + const release = uploadResponse.release; + switch (uploadResponse.result) { + case UploadReleaseResult.RELEASE_CREATED: + utils.logSuccess( + `uploaded new release ${release.displayVersion} (${release.buildVersion}) successfully!`, + ); + break; + case UploadReleaseResult.RELEASE_UPDATED: + utils.logSuccess( + `uploaded update to existing release ${release.displayVersion} (${release.buildVersion}) successfully!`, + ); + break; + case UploadReleaseResult.RELEASE_UNMODIFIED: + utils.logSuccess( + `re-uploaded already existing release ${release.displayVersion} (${release.buildVersion}) successfully!`, + ); + break; + default: + utils.logSuccess( + `uploaded release ${release.displayVersion} (${release.buildVersion}) successfully!`, + ); + } + utils.logSuccess(`View this release in the Firebase console: ${release.firebaseConsoleUri}`); + utils.logSuccess(`Share this release with testers who have access: ${release.testingUri}`); + utils.logSuccess( + `Download the release binary (link expires in 1 hour): ${release.binaryDownloadUri}`, + ); + releaseName = uploadResponse.release.name; + } catch (err: unknown) { + if (getErrStatus(err) === 404) { + throw new FirebaseError( + `App Distribution could not find your app ${appName}. ` + + `Make sure to onboard your app by pressing the "Get started" ` + + "button on the App Distribution page in the Firebase console: " + + "https://console.firebase.google.com/project/_/appdistribution", + { exit: 1 }, + ); + } + throw new FirebaseError(`Failed to upload release. ${getErrMsg(err)}`, { exit: 1 }); + } + + // If this is an app bundle and the certificate was originally blank fetch the updated + // certificate and print + if (aabInfo && !aabInfo.testCertificate) { + aabInfo = await requests.getAabInfo(appName); + if (aabInfo.testCertificate) { + utils.logBullet( + "After you upload an AAB for the first time, App Distribution " + + "generates a new test certificate. All AAB uploads are re-signed with this test " + + "certificate. Use the certificate fingerprints below to register your app " + + "signing key with API providers, such as Google Sign-In and Google Maps.\n" + + `MD-1 certificate fingerprint: ${aabInfo.testCertificate.hashMd5}\n` + + `SHA-1 certificate fingerprint: ${aabInfo.testCertificate.hashSha1}\n` + + `SHA-256 certificate fingerprint: ${aabInfo.testCertificate.hashSha256}`, + ); + } + } + + // Add release notes and distribute to testers/groups + await requests.updateReleaseNotes(releaseName, releaseNotes); + await requests.distribute(releaseName, testers, groups); + + // Run automated tests + if (testDevices.length) { + utils.logBullet("starting automated test (note: this feature is in beta)"); + const releaseTestPromises: Promise[] = []; + if (!testCases.length) { + // fallback to basic automated test + releaseTestPromises.push( + requests.createReleaseTest(releaseName, testDevices, loginCredential), + ); + } else { + for (const testCaseId of testCases) { + releaseTestPromises.push( + requests.createReleaseTest( + releaseName, + testDevices, + loginCredential, + `${appName}/testCases/${testCaseId}`, + ), + ); + } + } + const releaseTests = await Promise.all(releaseTestPromises); + utils.logSuccess(`${releaseTests.length} release test(s) started successfully`); + if (!testNonBlocking) { + await awaitTestResults(releaseTests, requests); + } + } +} + /** * Object representing an APK, AAB or IPA file. Used for uploading app distributions. */ @@ -58,3 +230,59 @@ export class Distribution { return this.fileName; } } + +async function awaitTestResults( + releaseTests: ReleaseTest[], + requests: AppDistributionClient, +): Promise { + const releaseTestNames = new Set(releaseTests.map((rt) => rt.name).filter((n): n is string => !!n)); + for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) { + utils.logBullet(`${releaseTestNames.size} automated test results are pending...`); + await delay(TEST_POLLING_INTERVAL_MILLIS); + for (const releaseTestName of releaseTestNames) { + const releaseTest = await requests.getReleaseTest(releaseTestName); + if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) { + releaseTestNames.delete(releaseTestName); + if (releaseTestNames.size === 0) { + utils.logSuccess("Automated test(s) passed!"); + return; + } else { + continue; + } + } + for (const execution of releaseTest.deviceExecutions) { + switch (execution.state) { + case "PASSED": + case "IN_PROGRESS": + continue; + case "FAILED": + throw new FirebaseError( + `Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`, + { exit: 1 }, + ); + case "INCONCLUSIVE": + throw new FirebaseError( + `Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`, + { exit: 1 }, + ); + default: + throw new FirebaseError( + `Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`, + { exit: 1 }, + ); + } + } + } + } + throw new FirebaseError("It took longer than expected to run your test(s), please try again.", { + exit: 1, + }); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function deviceToString(device: TestDevice): string { + return `${device.model} (${device.version}/${device.orientation}/${device.locale})`; +} diff --git a/src/appdistribution/options-parser-util.ts b/src/appdistribution/options-parser-util.ts index 1d8d83e5fcf..a4de257cb80 100644 --- a/src/appdistribution/options-parser-util.ts +++ b/src/appdistribution/options-parser-util.ts @@ -8,7 +8,7 @@ import { FieldHints, LoginCredential, TestDevice } from "./types"; * file and converts the input into an string[]. * Value takes precedent over file. */ -export function parseIntoStringArray(value: string, file: string): string[] { +export function parseIntoStringArray(value: string, file = ""): string[] { // If there is no value then the file gets parsed into a string to be split if (!value && file) { ensureFileExists(file); @@ -61,7 +61,10 @@ export function getAppName(options: any): string { if (!options.app) { throw new FirebaseError("set the --app option to a valid Firebase app id and try again"); } - const appId = options.app; + return toAppName(options.app); +} + +export function toAppName(appId: string) { return `projects/${appId.split(":")[1]}/apps/${appId}`; } @@ -70,7 +73,7 @@ export function getAppName(options: any): string { * and converts the input into a string[] of test device strings. * Value takes precedent over file. */ -export function parseTestDevices(value: string, file: string): TestDevice[] { +export function parseTestDevices(value: string, file = ""): TestDevice[] { // If there is no value then the file gets parsed into a string to be split if (!value && file) { ensureFileExists(file); diff --git a/src/commands/appdistribution-distribute.ts b/src/commands/appdistribution-distribute.ts index 61fb435fb2c..5d0346d2d0a 100644 --- a/src/commands/appdistribution-distribute.ts +++ b/src/commands/appdistribution-distribute.ts @@ -1,18 +1,9 @@ import * as fs from "fs-extra"; import { Command } from "../command"; -import * as utils from "../utils"; import { requireAuth } from "../requireAuth"; -import { AppDistributionClient } from "../appdistribution/client"; -import { - AabInfo, - IntegrationState, - UploadReleaseResult, - TestDevice, - ReleaseTest, -} from "../appdistribution/types"; -import { FirebaseError, getErrMsg, getErrStatus } from "../error"; -import { Distribution, DistributionFileType } from "../appdistribution/distribution"; +import { FirebaseError } from "../error"; +import { distribute, Distribution } from "../appdistribution/distribution"; import { ensureFileExists, getAppName, @@ -21,9 +12,6 @@ import { parseIntoStringArray, } from "../appdistribution/options-parser-util"; -const TEST_MAX_POLLING_RETRIES = 40; -const TEST_POLLING_INTERVAL_MILLIS = 30_000; - function getReleaseNotes(releaseNotes: string, releaseNotesFile: string): string { if (releaseNotes) { // Un-escape new lines from argument string @@ -104,206 +92,16 @@ export const command = new Command("appdistribution:distribute [] = []; - if (!testCases.length) { - // fallback to basic automated test - releaseTestPromises.push( - requests.createReleaseTest(releaseName, testDevices, loginCredential), - ); - } else { - for (const testCaseId of testCases) { - releaseTestPromises.push( - requests.createReleaseTest( - releaseName, - testDevices, - loginCredential, - `${appName}/testCases/${testCaseId}`, - ), - ); - } - } - const releaseTests = await Promise.all(releaseTestPromises); - utils.logSuccess(`${releaseTests.length} release test(s) started successfully`); - if (!options.testNonBlocking) { - await awaitTestResults(releaseTests, requests); - } - } - }); - -async function awaitTestResults( - releaseTests: ReleaseTest[], - requests: AppDistributionClient, -): Promise { - const releaseTestNames = new Set(releaseTests.map((rt) => rt.name!)); - for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) { - utils.logBullet(`${releaseTestNames.size} automated test results are pending...`); - await delay(TEST_POLLING_INTERVAL_MILLIS); - for (const releaseTestName of releaseTestNames) { - const releaseTest = await requests.getReleaseTest(releaseTestName); - if (releaseTest.deviceExecutions.every((e) => e.state === "PASSED")) { - releaseTestNames.delete(releaseTestName); - if (releaseTestNames.size === 0) { - utils.logSuccess("Automated test(s) passed!"); - return; - } else { - continue; - } - } - for (const execution of releaseTest.deviceExecutions) { - switch (execution.state) { - case "PASSED": - case "IN_PROGRESS": - continue; - case "FAILED": - throw new FirebaseError( - `Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`, - { exit: 1 }, - ); - case "INCONCLUSIVE": - throw new FirebaseError( - `Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`, - { exit: 1 }, - ); - default: - throw new FirebaseError( - `Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`, - { exit: 1 }, - ); - } - } - } - } - throw new FirebaseError("It took longer than expected to run your test(s), please try again.", { - exit: 1, + await distribute( + appName, + distribution, + testCases, + testDevices, + releaseNotes, + testers, + groups, + options.testNonBlocking, + loginCredential, + ); }); -} - -function delay(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function deviceToString(device: TestDevice): string { - return `${device.model} (${device.version}/${device.orientation}/${device.locale})`; -} diff --git a/src/mcp/prompts/index.ts b/src/mcp/prompts/index.ts index 83752cd9981..00886a6172a 100644 --- a/src/mcp/prompts/index.ts +++ b/src/mcp/prompts/index.ts @@ -14,6 +14,7 @@ const prompts: Record = { functions: [], remoteconfig: [], crashlytics: crashlyticsPrompts, + appdistribution: [], apphosting: [], database: [], }; diff --git a/src/mcp/tools/appdistribution/index.ts b/src/mcp/tools/appdistribution/index.ts new file mode 100644 index 00000000000..46d78af393f --- /dev/null +++ b/src/mcp/tools/appdistribution/index.ts @@ -0,0 +1,4 @@ +import type { ServerTool } from "../../tool"; +import { run_tests } from "./tests"; + +export const appdistributionTools: ServerTool[] = [run_tests]; diff --git a/src/mcp/tools/appdistribution/tests.ts b/src/mcp/tools/appdistribution/tests.ts new file mode 100644 index 00000000000..1fbfbc4942a --- /dev/null +++ b/src/mcp/tools/appdistribution/tests.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { ApplicationIdSchema } from "../../../crashlytics/filters"; +import { distribute, Distribution } from "../../../appdistribution/distribution"; +import { tool } from "../../tool"; +import { mcpError, toContent } from "../../util"; +import { + parseIntoStringArray, + parseTestDevices, + toAppName, +} from "../../../appdistribution/options-parser-util"; + +export const run_tests = tool( + { + name: "run_test", + description: `Upload an APK and run an existing test against it.`, + inputSchema: z.object({ + appId: ApplicationIdSchema, + releaseBinaryFile: z.string().describe("Path to the binary release (APK)."), + testDevices: z.string().describe( + `Semicolon-separated list of devices to run automated tests on, in the format + 'model=,version=,locale=,orientation='. Run 'gcloud firebase test android|ios models list' to see + available devices.`, + ), + testCaseIds: z.string().describe(`A comma-separated list of test case IDs.`), + }), + }, + async ({ appId, releaseBinaryFile, testDevices, testCaseIds }) => { + if (!appId) return mcpError(`Must specify 'appId' parameter.`); + if (!releaseBinaryFile) return mcpError(`Must specify 'releaseBinaryFile' parameter.`); + if (!testDevices) return mcpError(`Must specify 'testDevices' parameter.`); + if (!testCaseIds) return mcpError(`Must specify 'testCaseIds' parmeter.`); + + const appName = toAppName(appId); + const distribution = new Distribution(releaseBinaryFile); + const testCases = parseIntoStringArray(testCaseIds); + const devices = parseTestDevices(testDevices); + + return toContent(await distribute(appName, distribution, testCases, devices)); + }, +); diff --git a/src/mcp/tools/index.ts b/src/mcp/tools/index.ts index a6133d6c224..4c7db7a3852 100644 --- a/src/mcp/tools/index.ts +++ b/src/mcp/tools/index.ts @@ -9,6 +9,7 @@ import { messagingTools } from "./messaging/index"; import { remoteConfigTools } from "./remoteconfig/index"; import { crashlyticsTools } from "./crashlytics/index"; import { appHostingTools } from "./apphosting/index"; +import { appdistributionTools } from "./appdistribution/index"; import { realtimeDatabaseTools } from "./realtime_database/index"; import { functionsTools } from "./functions/index"; @@ -37,6 +38,7 @@ const tools: Record = { functions: addFeaturePrefix("functions", functionsTools), remoteconfig: addFeaturePrefix("remoteconfig", remoteConfigTools), crashlytics: addFeaturePrefix("crashlytics", crashlyticsTools), + appdistribution: addFeaturePrefix("appdistribution", appdistributionTools), apphosting: addFeaturePrefix("apphosting", appHostingTools), database: addFeaturePrefix("realtimedatabase", realtimeDatabaseTools), }; diff --git a/src/mcp/types.ts b/src/mcp/types.ts index 04381a744f8..5ae9e6daa83 100644 --- a/src/mcp/types.ts +++ b/src/mcp/types.ts @@ -12,6 +12,7 @@ export const SERVER_FEATURES = [ "functions", "remoteconfig", "crashlytics", + "appdistribution", "apphosting", "database", ] as const; diff --git a/src/mcp/util.ts b/src/mcp/util.ts index 92c3f382cdd..68e484b63c3 100644 --- a/src/mcp/util.ts +++ b/src/mcp/util.ts @@ -11,6 +11,7 @@ import { remoteConfigApiOrigin, storageOrigin, crashlyticsApiOrigin, + appDistributionOrigin, realtimeOrigin, } from "../api"; import { check } from "../ensureApiEnabled"; @@ -73,6 +74,7 @@ const SERVER_FEATURE_APIS: Record = { functions: functionsOrigin(), remoteconfig: remoteConfigApiOrigin(), crashlytics: crashlyticsApiOrigin(), + appdistribution: appDistributionOrigin(), apphosting: apphostingOrigin(), database: realtimeOrigin(), }; From 455d6d3f9c4388d1f7b62f50039f6b467dbe0a33 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Tue, 7 Oct 2025 12:29:10 -0400 Subject: [PATCH 03/19] Rename appdistribution directory to apptesting (#9268) * Rename appdistribution directory to apptesting * Make variables consistent with directory rename. --------- Co-authored-by: Jamie Rothfeder --- src/appdistribution/distribution.ts | 4 +++- src/mcp/prompts/index.ts | 2 +- src/mcp/tools/{appdistribution => apptesting}/index.ts | 2 +- src/mcp/tools/{appdistribution => apptesting}/tests.ts | 0 src/mcp/tools/index.ts | 4 ++-- src/mcp/types.ts | 2 +- src/mcp/util.ts | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) rename src/mcp/tools/{appdistribution => apptesting}/index.ts (57%) rename src/mcp/tools/{appdistribution => apptesting}/tests.ts (100%) diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index 30c46620965..eb3b5f5634b 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -235,7 +235,9 @@ async function awaitTestResults( releaseTests: ReleaseTest[], requests: AppDistributionClient, ): Promise { - const releaseTestNames = new Set(releaseTests.map((rt) => rt.name).filter((n): n is string => !!n)); + const releaseTestNames = new Set( + releaseTests.map((rt) => rt.name).filter((n): n is string => !!n), + ); for (let i = 0; i < TEST_MAX_POLLING_RETRIES; i++) { utils.logBullet(`${releaseTestNames.size} automated test results are pending...`); await delay(TEST_POLLING_INTERVAL_MILLIS); diff --git a/src/mcp/prompts/index.ts b/src/mcp/prompts/index.ts index a519d4803dd..345cc3538cf 100644 --- a/src/mcp/prompts/index.ts +++ b/src/mcp/prompts/index.ts @@ -14,7 +14,7 @@ const prompts: Record = { functions: [], remoteconfig: [], crashlytics: crashlyticsPrompts, - appdistribution: [], + apptesting: [], apphosting: [], database: [], }; diff --git a/src/mcp/tools/appdistribution/index.ts b/src/mcp/tools/apptesting/index.ts similarity index 57% rename from src/mcp/tools/appdistribution/index.ts rename to src/mcp/tools/apptesting/index.ts index 46d78af393f..b40f863e12e 100644 --- a/src/mcp/tools/appdistribution/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,4 +1,4 @@ import type { ServerTool } from "../../tool"; import { run_tests } from "./tests"; -export const appdistributionTools: ServerTool[] = [run_tests]; +export const apptestingTools: ServerTool[] = [run_tests]; diff --git a/src/mcp/tools/appdistribution/tests.ts b/src/mcp/tools/apptesting/tests.ts similarity index 100% rename from src/mcp/tools/appdistribution/tests.ts rename to src/mcp/tools/apptesting/tests.ts diff --git a/src/mcp/tools/index.ts b/src/mcp/tools/index.ts index e36de6eeb8a..3898e3f71d4 100644 --- a/src/mcp/tools/index.ts +++ b/src/mcp/tools/index.ts @@ -9,7 +9,7 @@ import { messagingTools } from "./messaging/index"; import { remoteConfigTools } from "./remoteconfig/index"; import { crashlyticsTools } from "./crashlytics/index"; import { appHostingTools } from "./apphosting/index"; -import { appdistributionTools } from "./appdistribution/index"; +import { apptestingTools } from "./apptesting/index"; import { realtimeDatabaseTools } from "./realtime_database/index"; import { functionsTools } from "./functions/index"; @@ -38,7 +38,7 @@ const tools: Record = { functions: addFeaturePrefix("functions", functionsTools), remoteconfig: addFeaturePrefix("remoteconfig", remoteConfigTools), crashlytics: addFeaturePrefix("crashlytics", crashlyticsTools), - appdistribution: addFeaturePrefix("appdistribution", appdistributionTools), + apptesting: addFeaturePrefix("apptesting", apptestingTools), apphosting: addFeaturePrefix("apphosting", appHostingTools), database: addFeaturePrefix("realtimedatabase", realtimeDatabaseTools), }; diff --git a/src/mcp/types.ts b/src/mcp/types.ts index b4321bf5494..19dbcd1577d 100644 --- a/src/mcp/types.ts +++ b/src/mcp/types.ts @@ -12,7 +12,7 @@ export const SERVER_FEATURES = [ "functions", "remoteconfig", "crashlytics", - "appdistribution", + "apptesting", "apphosting", "database", ] as const; diff --git a/src/mcp/util.ts b/src/mcp/util.ts index 68e484b63c3..5a38690494e 100644 --- a/src/mcp/util.ts +++ b/src/mcp/util.ts @@ -74,7 +74,7 @@ const SERVER_FEATURE_APIS: Record = { functions: functionsOrigin(), remoteconfig: remoteConfigApiOrigin(), crashlytics: crashlyticsApiOrigin(), - appdistribution: appDistributionOrigin(), + apptesting: appDistributionOrigin(), apphosting: apphostingOrigin(), database: realtimeOrigin(), }; From 1744e5ea85248d1e8243fb2beae6b65de0299bea Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Wed, 8 Oct 2025 10:20:34 -0400 Subject: [PATCH 04/19] Use a datastructure to represent test devices rather than a string. (#9280) * Use a datastructure to represent test devices rather than a string. * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Pretty --------- Co-authored-by: Jamie Rothfeder Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/mcp/tools/apptesting/tests.ts | 60 +++++++++++++++++++------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 1fbfbc4942a..ff0656ebebb 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -1,40 +1,54 @@ import { z } from "zod"; import { ApplicationIdSchema } from "../../../crashlytics/filters"; import { distribute, Distribution } from "../../../appdistribution/distribution"; + import { tool } from "../../tool"; -import { mcpError, toContent } from "../../util"; -import { - parseIntoStringArray, - parseTestDevices, - toAppName, -} from "../../../appdistribution/options-parser-util"; +import { toContent } from "../../util"; +import { parseIntoStringArray, toAppName } from "../../../appdistribution/options-parser-util"; + +const TestDeviceSchema = z + .object({ + model: z.string(), + version: z.string(), + locale: z.string(), + orientation: z.enum(["portrait", "landscape"]), + }) + .describe( + `Device to run automated test on. Can run 'gcloud firebase test android|ios models list' to see available devices.`, + ); export const run_tests = tool( { name: "run_test", - description: `Upload an APK and run an existing test against it.`, + description: "Upload a binary and run automated tests.", inputSchema: z.object({ appId: ApplicationIdSchema, releaseBinaryFile: z.string().describe("Path to the binary release (APK)."), - testDevices: z.string().describe( - `Semicolon-separated list of devices to run automated tests on, in the format - 'model=,version=,locale=,orientation='. Run 'gcloud firebase test android|ios models list' to see - available devices.`, - ), + testDevices: z.array(TestDeviceSchema).default([ + { + model: "tokay", + version: "36", + locale: "en", + orientation: "portrait", + }, + { + model: "e1q", + version: "34", + locale: "en", + orientation: "portrait", + }, + ]), testCaseIds: z.string().describe(`A comma-separated list of test case IDs.`), }), }, async ({ appId, releaseBinaryFile, testDevices, testCaseIds }) => { - if (!appId) return mcpError(`Must specify 'appId' parameter.`); - if (!releaseBinaryFile) return mcpError(`Must specify 'releaseBinaryFile' parameter.`); - if (!testDevices) return mcpError(`Must specify 'testDevices' parameter.`); - if (!testCaseIds) return mcpError(`Must specify 'testCaseIds' parmeter.`); - - const appName = toAppName(appId); - const distribution = new Distribution(releaseBinaryFile); - const testCases = parseIntoStringArray(testCaseIds); - const devices = parseTestDevices(testDevices); - - return toContent(await distribute(appName, distribution, testCases, devices)); + return toContent( + await distribute( + toAppName(appId), + new Distribution(releaseBinaryFile), + parseIntoStringArray(testCaseIds), + testDevices, + ), + ); }, ); From 81c1a61f9d871276363367e90d54799104e7aebd Mon Sep 17 00:00:00 2001 From: tagboola Date: Thu, 9 Oct 2025 16:00:44 -0400 Subject: [PATCH 05/19] Add run_test prompt (#9292) * Add initial MCP prompt for running automated tests * Fix typos --- src/mcp/prompts/apptesting/index.ts | 9 ++ src/mcp/prompts/apptesting/run_test.ts | 116 +++++++++++++++++++++++++ src/mcp/prompts/index.ts | 3 +- 3 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 src/mcp/prompts/apptesting/index.ts create mode 100644 src/mcp/prompts/apptesting/run_test.ts diff --git a/src/mcp/prompts/apptesting/index.ts b/src/mcp/prompts/apptesting/index.ts new file mode 100644 index 00000000000..5dc7acbb9b3 --- /dev/null +++ b/src/mcp/prompts/apptesting/index.ts @@ -0,0 +1,9 @@ +import { isEnabled } from "../../../experiments"; +import { runTest } from "./run_test"; +import type { ServerPrompt } from "../../prompt"; + +export const apptestingPrompts: ServerPrompt[] = []; + +if (isEnabled("mcpalpha")) { + apptestingPrompts.push(runTest); +} diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts new file mode 100644 index 00000000000..451685df558 --- /dev/null +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -0,0 +1,116 @@ +import { prompt } from "../../prompt"; + +export const runTest = prompt( + { + name: "run_test", + description: "Run a test with the Firebase App Testing agent", + omitPrefix: false, + arguments: [ + { + name: "testDescription", + description: + "Description of the test you want to run. The agent will use the description to generate a test case that will be used as input for the AI-guided test", + required: false, + }, + ], + annotations: { + title: "Run an App Testing AI-guided test", + }, + }, + async ({ testDescription }, { accountEmail, projectId }) => { + return [ + { + role: "user" as const, + content: { + type: "text", + text: ` +You are going to help a developer run a test for their mobile app +using the Firebase App Testing agent. + +Active user: ${accountEmail || ""} +Project ID: ${projectId || ""} + +## Prerequisites + +Here are a list of prerequisite steps that must be completed before running a test. + +1. **Make sure this is an Android app**. The App Testing agent only works with Android apps. If + this is not an Android app, instruct the user that this command can't be used with this app. + +2. **Make sure the user is logged in. No App Testing tools will work if the user is not logged in.** + a. Use the \`firebase_get_environment\` tool to verify that the user is logged in. + b. If the Firebase 'Active user' is set to , instruct the user to run \`firebase login\` + before continuing. Ignore other fields that are set to . We are just making sure the + user is logged in. + +3. **Get the Firebase app ID.** + + The \`firebase_get_environment\` tool should return a list of detected app IDs, where the app + ID contains four colon (":") delimited parts: a version number (typically "1"), a project + number, a platform type ("android", "ios", or "web"). Ask the user confirm if there is only + a single app ID, or to choose one if there are multiple app IDs. + + If the tool does not return a list of detected apps, just ask the user for it. + +4. **Confirm that the application ID of the app matches the bundle ID of the Firebase app** + + The \`firebase_get_environment\` tool returns a list of detected app IDs mapped to their corresponding + bundle IDs. If the developer selected an app ID from the the list of detected app IDs, this already + confirms that the bundle ID matches the app ID. If not, get the application IDs of all the variants of + the app. Then get the bundle ID of the Firebase app by calling the \`firebase_list_apps\` tool and + confirming that the \`namespace\` field of the app with the selected app ID matches one of the application + IDs of the variants. + +## Test Case Generation + + Once you have completed the required steps, you need the help the user generate a "test case", which is the input to the + app testing agent. A test case consists of multiple steps where each step contains the following fields: + + * Goal (required): In one sentence or less, describe what you want the agent to do in this step. + * Hint (optional): Provide additional information to help Gemini understand and navigate your app. + * Success Criteria (optional): Your success criteria should be phrased as an observation, such as 'The screen shows a + success message' or 'The checkout page is visible'. + + The developer has optionally specified the following description for their test: + * ${testDescription} + + Sometimes, test descriptions that developers provide tend to be too vague and lack the necessary details for the + app testing agent to be able to reliably re-run the tests with consistent results. Test cases should follow these + guidelines to ensure that they are structured in a way to make the agent more reliable. + + * Prefer multiple steps with smaller, detailed goals. Broader, more open-ended goals can lead to unreliable tests + since the app testing agent can more easily veer of course. It should only take a few actions to accomplish a goal. + For example, if a step has a list in it, it should probably be broken up into multiple steps. Steps do not need + to be too small though. The test case should provide a good balance between strict guidance and flexibility. As a + rule of thumb, each step should require between 2-5 actions. + * Include a hint and success criteria whenever possible. Specifically, try to always include a success criteria to help + the agent determine when the goal has been completed. + * Avoid functionality that the app testing agent struggles with. The app testing agent struggles with the following: + * Journeys that require specific timing (like observing that something should be visible for a certain number of + seconds), interacting with moving or transient elements, etc. + * Playing games or generally interacting with drawn visuals that would require pixel input + * Complex swipe interactions, multi-finger gestures, etc., which aren't supported + + First, analyze the code to get an understanding of how the app works. Get all the available screens in the app and the + different actions for each screen. Understand what functionality is and isn't available to the app testing agent. + Only include specific details in the test case if you are certain they will be available to the agent, otherwise the + agent will likely fail if it tries to follow specific guidance that doesn't work (e.g. click the 'Play' button but the + button isn't visible to the app testing agent). Do not include Android resource ids in the test case. Include + explanations that prove that each step includes between 2-5 actions. Using that information as context and the guidelines + above, convert the test description provided by the user to make it easier for the agent to follow so that the tests can + be re-run reliably. Generate an explanation on why you generated the new test case the way you did, and then generate the + new test case, which again is an array of steps where each step contains a goal, hint, and success criteria. Show this + to the user and have them confirm before moving forward. + +## Run Test + + Use the apptesting_run_test tool to run an automated test with the following as input: + * The generated test case that as been confirmed by the user + * An APK. If there is no APK present, build the app to produce one. Make sure to build the variant of the app + with the same bundle ID as the Firebase app. +`.trim(), + }, + }, + ]; + }, +); diff --git a/src/mcp/prompts/index.ts b/src/mcp/prompts/index.ts index 40da91632c5..ceea492ddd9 100644 --- a/src/mcp/prompts/index.ts +++ b/src/mcp/prompts/index.ts @@ -3,6 +3,7 @@ import { ServerPrompt } from "../prompt"; import { corePrompts } from "./core"; import { dataconnectPrompts } from "./dataconnect"; import { crashlyticsPrompts } from "./crashlytics"; +import { apptestingPrompts } from "./apptesting"; const prompts: Record = { core: corePrompts, @@ -14,7 +15,7 @@ const prompts: Record = { functions: [], remoteconfig: [], crashlytics: crashlyticsPrompts, - apptesting: [], + apptesting: apptestingPrompts, apphosting: [], database: [], }; From b3098073927958f6428050e07f13ca3cf78cd332 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Thu, 16 Oct 2025 09:32:49 -0400 Subject: [PATCH 06/19] MCP tool `apptesting_run_test` can create and run a on-off test. (#9321) * Create a on-off test and execute. * Can now create a on-off test. * Update src/appdistribution/client.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * Update src/mcp/tools/apptesting/tests.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * PR feedback * Separate test check in to a different tool so that gemini can orchestrate running and checking for completion. * Set the devices field to optional --------- Co-authored-by: Jamie Rothfeder Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- src/appdistribution/client.ts | 3 + src/appdistribution/distribution.ts | 130 ++------------------ src/appdistribution/types.ts | 11 ++ src/commands/appdistribution-distribute.ts | 131 ++++++++++++++++++++- src/mcp/tools/apptesting/index.ts | 4 +- src/mcp/tools/apptesting/tests.ts | 95 ++++++++++----- 6 files changed, 221 insertions(+), 153 deletions(-) diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 839e3d8bdcc..fc3872f00e2 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -9,6 +9,7 @@ import { appDistributionOrigin } from "../api"; import { AabInfo, + AIInstruction, BatchRemoveTestersResponse, Group, ListGroupsResponse, @@ -280,6 +281,7 @@ export class AppDistributionClient { async createReleaseTest( releaseName: string, devices: TestDevice[], + aiInstruction?: AIInstruction, loginCredential?: LoginCredential, testCaseName?: string, ): Promise { @@ -291,6 +293,7 @@ export class AppDistributionClient { deviceExecutions: devices.map(mapDeviceToExecution), loginCredential, testCase: testCaseName, + aiInstructions: aiInstruction, }, }); return response.body; diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index eb3b5f5634b..883f225a346 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -2,88 +2,25 @@ import * as fs from "fs-extra"; import { logger } from "../logger"; import * as pathUtil from "path"; import * as utils from "../utils"; -import { - AabInfo, - IntegrationState, - UploadReleaseResult, - TestDevice, - ReleaseTest, - LoginCredential, -} from "../appdistribution/types"; +import { UploadReleaseResult, TestDevice, ReleaseTest } from "../appdistribution/types"; import { AppDistributionClient } from "./client"; import { FirebaseError, getErrMsg, getErrStatus } from "../error"; +const TEST_MAX_POLLING_RETRIES = 40; +const TEST_POLLING_INTERVAL_MILLIS = 30_000; + export enum DistributionFileType { IPA = "ipa", APK = "apk", AAB = "aab", } -const TEST_MAX_POLLING_RETRIES = 40; -const TEST_POLLING_INTERVAL_MILLIS = 30_000; - -/** - * Execute an app distribution action - */ -export async function distribute( +export async function upload( + requests: AppDistributionClient, appName: string, distribution: Distribution, - testCases: string[], - testDevices: TestDevice[], - releaseNotes?: string, - testers?: string[], - groups?: string[], - testNonBlocking?: boolean, - loginCredential?: LoginCredential, -) { - const requests = new AppDistributionClient(); - let aabInfo: AabInfo | undefined; - if (distribution.distributionFileType() === DistributionFileType.AAB) { - try { - aabInfo = await requests.getAabInfo(appName); - } catch (err: unknown) { - if (getErrStatus(err) === 404) { - throw new FirebaseError( - `App Distribution could not find your app ${appName}. ` + - `Make sure to onboard your app by pressing the "Get started" ` + - "button on the App Distribution page in the Firebase console: " + - "https://console.firebase.google.com/project/_/appdistribution", - { exit: 1 }, - ); - } - throw new FirebaseError(`failed to determine AAB info. ${getErrMsg(err)}`, { exit: 1 }); - } - - if ( - aabInfo.integrationState !== IntegrationState.INTEGRATED && - aabInfo.integrationState !== IntegrationState.AAB_STATE_UNAVAILABLE - ) { - switch (aabInfo.integrationState) { - case IntegrationState.PLAY_ACCOUNT_NOT_LINKED: { - throw new FirebaseError("This project is not linked to a Google Play account."); - } - case IntegrationState.APP_NOT_PUBLISHED: { - throw new FirebaseError('"This app is not published in the Google Play console.'); - } - case IntegrationState.NO_APP_WITH_GIVEN_BUNDLE_ID_IN_PLAY_ACCOUNT: { - throw new FirebaseError("App with matching package name does not exist in Google Play."); - } - case IntegrationState.PLAY_IAS_TERMS_NOT_ACCEPTED: { - throw new FirebaseError( - "You must accept the Play Internal App Sharing (IAS) terms to upload AABs.", - ); - } - default: { - throw new FirebaseError( - "App Distribution failed to process the AAB: " + aabInfo.integrationState, - ); - } - } - } - } - +): Promise { utils.logBullet("uploading binary..."); - let releaseName; try { const operationName = await requests.uploadRelease(appName, distribution); @@ -117,7 +54,7 @@ export async function distribute( utils.logSuccess( `Download the release binary (link expires in 1 hour): ${release.binaryDownloadUri}`, ); - releaseName = uploadResponse.release.name; + return uploadResponse.release.name; } catch (err: unknown) { if (getErrStatus(err) === 404) { throw new FirebaseError( @@ -130,55 +67,6 @@ export async function distribute( } throw new FirebaseError(`Failed to upload release. ${getErrMsg(err)}`, { exit: 1 }); } - - // If this is an app bundle and the certificate was originally blank fetch the updated - // certificate and print - if (aabInfo && !aabInfo.testCertificate) { - aabInfo = await requests.getAabInfo(appName); - if (aabInfo.testCertificate) { - utils.logBullet( - "After you upload an AAB for the first time, App Distribution " + - "generates a new test certificate. All AAB uploads are re-signed with this test " + - "certificate. Use the certificate fingerprints below to register your app " + - "signing key with API providers, such as Google Sign-In and Google Maps.\n" + - `MD-1 certificate fingerprint: ${aabInfo.testCertificate.hashMd5}\n` + - `SHA-1 certificate fingerprint: ${aabInfo.testCertificate.hashSha1}\n` + - `SHA-256 certificate fingerprint: ${aabInfo.testCertificate.hashSha256}`, - ); - } - } - - // Add release notes and distribute to testers/groups - await requests.updateReleaseNotes(releaseName, releaseNotes); - await requests.distribute(releaseName, testers, groups); - - // Run automated tests - if (testDevices.length) { - utils.logBullet("starting automated test (note: this feature is in beta)"); - const releaseTestPromises: Promise[] = []; - if (!testCases.length) { - // fallback to basic automated test - releaseTestPromises.push( - requests.createReleaseTest(releaseName, testDevices, loginCredential), - ); - } else { - for (const testCaseId of testCases) { - releaseTestPromises.push( - requests.createReleaseTest( - releaseName, - testDevices, - loginCredential, - `${appName}/testCases/${testCaseId}`, - ), - ); - } - } - const releaseTests = await Promise.all(releaseTestPromises); - utils.logSuccess(`${releaseTests.length} release test(s) started successfully`); - if (!testNonBlocking) { - await awaitTestResults(releaseTests, requests); - } - } } /** @@ -231,7 +119,7 @@ export class Distribution { } } -async function awaitTestResults( +export async function awaitTestResults( releaseTests: ReleaseTest[], requests: AppDistributionClient, ): Promise { diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index f90155a4cd2..1a674d44d1f 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -127,4 +127,15 @@ export interface ReleaseTest { deviceExecutions: DeviceExecution[]; loginCredential?: LoginCredential; testCase?: string; + aiInstructions?: AIInstruction; +} + +export interface AIInstruction { + steps: AIStep[]; +} + +export interface AIStep { + goal: string; + hint?: string; + successCriteria?: string; } diff --git a/src/commands/appdistribution-distribute.ts b/src/commands/appdistribution-distribute.ts index 5d0346d2d0a..d13da419856 100644 --- a/src/commands/appdistribution-distribute.ts +++ b/src/commands/appdistribution-distribute.ts @@ -2,8 +2,13 @@ import * as fs from "fs-extra"; import { Command } from "../command"; import { requireAuth } from "../requireAuth"; -import { FirebaseError } from "../error"; -import { distribute, Distribution } from "../appdistribution/distribution"; +import { FirebaseError, getErrMsg, getErrStatus } from "../error"; +import { + awaitTestResults, + Distribution, + DistributionFileType, + upload, +} from "../appdistribution/distribution"; import { ensureFileExists, getAppName, @@ -11,6 +16,15 @@ import { parseTestDevices, parseIntoStringArray, } from "../appdistribution/options-parser-util"; +import { + AabInfo, + IntegrationState, + LoginCredential, + ReleaseTest, + TestDevice, +} from "../appdistribution/types"; +import { AppDistributionClient } from "../appdistribution/client"; +import * as utils from "../utils"; function getReleaseNotes(releaseNotes: string, releaseNotesFile: string): string { if (releaseNotes) { @@ -105,3 +119,116 @@ export const command = new Command("appdistribution:distribute [] = []; + if (!testCases.length) { + // fallback to basic automated test + releaseTestPromises.push( + requests.createReleaseTest(releaseName, testDevices, undefined, loginCredential), + ); + } else { + for (const testCaseId of testCases) { + releaseTestPromises.push( + requests.createReleaseTest( + releaseName, + testDevices, + undefined, + loginCredential, + `${appName}/testCases/${testCaseId}`, + ), + ); + } + } + const releaseTests = await Promise.all(releaseTestPromises); + utils.logSuccess(`${releaseTests.length} release test(s) started successfully`); + if (!testNonBlocking) { + await awaitTestResults(releaseTests, requests); + } + } +} diff --git a/src/mcp/tools/apptesting/index.ts b/src/mcp/tools/apptesting/index.ts index b40f863e12e..22592e537ce 100644 --- a/src/mcp/tools/apptesting/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,4 +1,4 @@ import type { ServerTool } from "../../tool"; -import { run_tests } from "./tests"; +import { check_test, run_tests } from "./tests"; -export const apptestingTools: ServerTool[] = [run_tests]; +export const apptestingTools: ServerTool[] = [run_tests, check_test]; diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index ff0656ebebb..da1207df4d6 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -1,54 +1,93 @@ import { z } from "zod"; import { ApplicationIdSchema } from "../../../crashlytics/filters"; -import { distribute, Distribution } from "../../../appdistribution/distribution"; +import { upload, Distribution } from "../../../appdistribution/distribution"; import { tool } from "../../tool"; import { toContent } from "../../util"; -import { parseIntoStringArray, toAppName } from "../../../appdistribution/options-parser-util"; +import { toAppName } from "../../../appdistribution/options-parser-util"; +import { AppDistributionClient } from "../../../appdistribution/client"; const TestDeviceSchema = z .object({ model: z.string(), version: z.string(), locale: z.string(), - orientation: z.enum(["portrait", "landscape"]), + orientation: z.string(), }) .describe( `Device to run automated test on. Can run 'gcloud firebase test android|ios models list' to see available devices.`, ); +const AIStepSchema = z + .object({ + goal: z.string().describe("A goal to be accomplished during the test."), + hint: z + .string() + .optional() + .describe("Hint text containing suggestions to help the agent accomplish the goal."), + successCriteria: z + .string() + .optional() + .describe( + "A description of criteria the agent should use to determine if the goal has been successfully completed.", + ), + }) + .describe("Step within a test case; run during the execution of the test."); + export const run_tests = tool( { name: "run_test", - description: "Upload a binary and run automated tests.", + description: `Run a remote test.`, inputSchema: z.object({ appId: ApplicationIdSchema, releaseBinaryFile: z.string().describe("Path to the binary release (APK)."), - testDevices: z.array(TestDeviceSchema).default([ - { - model: "tokay", - version: "36", - locale: "en", - orientation: "portrait", - }, - { - model: "e1q", - version: "34", - locale: "en", - orientation: "portrait", - }, - ]), - testCaseIds: z.string().describe(`A comma-separated list of test case IDs.`), + testDevices: z + .array(TestDeviceSchema) + .optional() + .default([ + { + model: "tokay", + version: "36", + locale: "en", + orientation: "portrait", + }, + { + model: "e1q", + version: "34", + locale: "en", + orientation: "portrait", + }, + ]), + testCase: z + .array(AIStepSchema) + .describe("Test case containing the steps that are run during its execution."), }), + annotations: { + title: "Run a Remote Test", + readOnlyHint: false, + }, }, - async ({ appId, releaseBinaryFile, testDevices, testCaseIds }) => { - return toContent( - await distribute( - toAppName(appId), - new Distribution(releaseBinaryFile), - parseIntoStringArray(testCaseIds), - testDevices, - ), - ); + async ({ appId, releaseBinaryFile, testDevices, testCase }) => { + const client = new AppDistributionClient(); + const releaeName = await upload(client, toAppName(appId), new Distribution(releaseBinaryFile)); + return toContent(await client.createReleaseTest(releaeName, testDevices, { steps: testCase })); + }, +); + +export const check_test = tool( + { + name: "check_test", + description: "Check the status of a remote test.", + inputSchema: z.object({ + name: z.string().describe("The name of the release test returned by the run_test tool."), + }), + annotations: { + title: "Check Remote Test", + readOnlyHint: true, + }, + }, + async ({ name }) => { + const client = new AppDistributionClient(); + return toContent(await client.getReleaseTest(name)); }, ); From 973e8d7ccf7cdb42b41ec6d5d3f10ccf270a1f22 Mon Sep 17 00:00:00 2001 From: tagboola Date: Thu, 16 Oct 2025 10:18:14 -0400 Subject: [PATCH 07/19] Use the same default device that's used in the Console (#9320) --- src/mcp/tools/apptesting/tests.ts | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index da1207df4d6..0030205b292 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -41,23 +41,14 @@ export const run_tests = tool( inputSchema: z.object({ appId: ApplicationIdSchema, releaseBinaryFile: z.string().describe("Path to the binary release (APK)."), - testDevices: z - .array(TestDeviceSchema) - .optional() - .default([ - { - model: "tokay", - version: "36", - locale: "en", - orientation: "portrait", - }, - { - model: "e1q", - version: "34", - locale: "en", - orientation: "portrait", - }, - ]), + testDevices: z.array(TestDeviceSchema).default([ + { + model: "MediumPhone.arm", + version: "30", + locale: "en_US", + orientation: "portrait", + }, + ]), testCase: z .array(AIStepSchema) .describe("Test case containing the steps that are run during its execution."), From 82e17249517a90eaeb0a7a969f71a3069e19ebb6 Mon Sep 17 00:00:00 2001 From: tagboola Date: Thu, 16 Oct 2025 10:19:47 -0400 Subject: [PATCH 08/19] Update prompt to support generating a test case when there is no test description (#9322) * Use the same default device that's used in the Console * Update prompt to support generating a test case when there is no test description passed --- src/mcp/prompts/apptesting/run_test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 451685df558..847463b33b1 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -98,7 +98,8 @@ Here are a list of prerequisite steps that must be completed before running a te button isn't visible to the app testing agent). Do not include Android resource ids in the test case. Include explanations that prove that each step includes between 2-5 actions. Using that information as context and the guidelines above, convert the test description provided by the user to make it easier for the agent to follow so that the tests can - be re-run reliably. Generate an explanation on why you generated the new test case the way you did, and then generate the + be re-run reliably. If there is no test description, generate a test case that you think will be useful given the functionality + of the app. Generate an explanation on why you generated the new test case the way you did, and then generate the new test case, which again is an array of steps where each step contains a goal, hint, and success criteria. Show this to the user and have them confirm before moving forward. From 380b001d2911f4b2934f576e8993f624903f7d94 Mon Sep 17 00:00:00 2001 From: tagboola Date: Thu, 23 Oct 2025 15:50:34 -0400 Subject: [PATCH 09/19] Add custom auto-enablement for app testing (#9373) * Add custom auto-enablement for app testing * Address gemini code assist comments * Fix intersection bug * Fix issues with test --- src/mcp/prompts/apptesting/run_test.ts | 1 + src/mcp/tools/apptesting/tests.ts | 2 + src/mcp/util.ts | 1 + src/mcp/util/apptesting/availability.spec.ts | 102 +++++++++++++++++++ src/mcp/util/apptesting/availability.ts | 33 ++++++ src/mcp/util/availability.spec.ts | 2 +- src/mcp/util/availability.ts | 2 + 7 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/mcp/util/apptesting/availability.spec.ts create mode 100644 src/mcp/util/apptesting/availability.ts diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 847463b33b1..536bd6122b3 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -1,6 +1,7 @@ import { prompt } from "../../prompt"; export const runTest = prompt( + "apptesting", { name: "run_test", description: "Run a test with the Firebase App Testing agent", diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 0030205b292..fd298adbadd 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -35,6 +35,7 @@ const AIStepSchema = z .describe("Step within a test case; run during the execution of the test."); export const run_tests = tool( + "apptesting", { name: "run_test", description: `Run a remote test.`, @@ -66,6 +67,7 @@ export const run_tests = tool( ); export const check_test = tool( + "apptesting", { name: "check_test", description: "Check the status of a remote test.", diff --git a/src/mcp/util.ts b/src/mcp/util.ts index 5aa5a5a4e05..f7a11b88d2a 100644 --- a/src/mcp/util.ts +++ b/src/mcp/util.ts @@ -89,6 +89,7 @@ const DETECTED_API_FEATURES: Record = { functions: undefined, remoteconfig: undefined, crashlytics: undefined, + apptesting: undefined, apphosting: undefined, database: undefined, }; diff --git a/src/mcp/util/apptesting/availability.spec.ts b/src/mcp/util/apptesting/availability.spec.ts new file mode 100644 index 00000000000..9485eac2af9 --- /dev/null +++ b/src/mcp/util/apptesting/availability.spec.ts @@ -0,0 +1,102 @@ +import * as mockfs from "mock-fs"; +import * as sinon from "sinon"; +import { FirebaseMcpServer } from "../../index"; +import { RC } from "../../../rc"; +import { Config } from "../../../config"; +import { McpContext } from "../../types"; +import { isAppTestingAvailable } from "./availability"; +import { expect } from "chai"; +import * as ensureApiEnabled from "../../../ensureApiEnabled"; + +describe("isAppTestingAvailable", () => { + let sandbox: sinon.SinonSandbox; + let checkStub: sinon.SinonStub; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + checkStub = sandbox.stub(ensureApiEnabled, "check"); + }); + + afterEach(() => { + sandbox.restore(); + mockfs.restore(); + }); + + const mockContext = (projectDir: string): McpContext => ({ + projectId: "test-project", + accountEmail: null, + config: { + projectDir: projectDir, + } as Config, + host: new FirebaseMcpServer({}), + rc: {} as RC, + firebaseCliCommand: "firebase", + }); + + it("returns false for non mobile project", async () => { + checkStub.resolves(true); + mockfs({ + "/test-dir": { + "package.json": '{ "name": "web-app" }', + "index.html": "", + }, + }); + const result = await isAppTestingAvailable(mockContext("/test-dir")); + expect(result).to.be.false; + }); + + it("returns false if App Distribution API isn't enabled", async () => { + checkStub.resolves(false); + mockfs({ + "/test-dir": { + android: { + "build.gradle": "", + src: { main: {} }, + }, + }, + }); + const result = await isAppTestingAvailable(mockContext("/test-dir")); + expect(result).to.be.false; + }); + + it("returns true for an Android project with API enabled", async () => { + checkStub.resolves(true); + mockfs({ + "/test-dir": { + android: { + "build.gradle": "", + src: { main: {} }, + }, + }, + }); + const result = await isAppTestingAvailable(mockContext("/test-dir")); + expect(result).to.be.true; + }); + + it("returns true for an iOS project with API enabled", async () => { + checkStub.resolves(true); + mockfs({ + "/test-dir": { + ios: { + Podfile: "", + "Project.xcodeproj": {}, + }, + }, + }); + const result = await isAppTestingAvailable(mockContext("/test-dir")); + expect(result).to.be.true; + }); + + it("returns true for an Flutter project with API enabled", async () => { + checkStub.resolves(true); + mockfs({ + "/test-dir": { + "pubspec.yaml": "", + ios: { "Runner.xcodeproj": {} }, + android: { src: { main: {} } }, + }, + }); + const result = await isAppTestingAvailable(mockContext("/test-dir")); + expect(result).to.be.true; + }); +}); diff --git a/src/mcp/util/apptesting/availability.ts b/src/mcp/util/apptesting/availability.ts new file mode 100644 index 00000000000..62475bf534a --- /dev/null +++ b/src/mcp/util/apptesting/availability.ts @@ -0,0 +1,33 @@ +import { appDistributionOrigin } from "../../../api"; +import { getPlatformsFromFolder, Platform } from "../../../appUtils"; +import { check } from "../../../ensureApiEnabled"; +import { timeoutFallback } from "../../../timeout"; +import { McpContext } from "../../types"; + +/** + * Returns whether or not App Testing should be enabled + */ +export async function isAppTestingAvailable(ctx: McpContext): Promise { + const host = ctx.host; + const projectDir = ctx.config.projectDir; + const platforms = await getPlatformsFromFolder(projectDir); + + const supportedPlatforms = [Platform.FLUTTER, Platform.ANDROID, Platform.IOS]; + + if (!platforms.some((p) => supportedPlatforms.includes(p))) { + host.log("debug", `Found no supported App Testing platforms.`); + return false; + } + + // Checkf if App Distribution API is active + try { + return await timeoutFallback( + check(ctx.projectId, appDistributionOrigin(), "", true), + true, + 3000, + ); + } catch (e) { + // If there was a network error, default to enabling the feature + return true; + } +} diff --git a/src/mcp/util/availability.spec.ts b/src/mcp/util/availability.spec.ts index 4b43a92a0be..2f42118117c 100644 --- a/src/mcp/util/availability.spec.ts +++ b/src/mcp/util/availability.spec.ts @@ -42,7 +42,7 @@ describe("getDefaultFeatureAvailabilityCheck", () => { // Test all other features that rely on checkFeatureActive const featuresThatUseCheckActive = SERVER_FEATURES.filter( - (f) => f !== "core" && f !== "crashlytics", + (f) => f !== "core" && f !== "crashlytics" && f !== "apptesting", ); for (const feature of featuresThatUseCheckActive) { diff --git a/src/mcp/util/availability.ts b/src/mcp/util/availability.ts index ce96e5f8e14..f5f487fb812 100644 --- a/src/mcp/util/availability.ts +++ b/src/mcp/util/availability.ts @@ -1,6 +1,7 @@ import { McpContext, ServerFeature } from "../types"; import { checkFeatureActive } from "../util"; import { isCrashlyticsAvailable } from "./crashlytics/availability"; +import { isAppTestingAvailable } from "./apptesting/availability"; const DEFAULT_AVAILABILITY_CHECKS: Record Promise> = { // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -22,6 +23,7 @@ const DEFAULT_AVAILABILITY_CHECKS: Record Pr crashlytics: isCrashlyticsAvailable, apphosting: (ctx: McpContext): Promise => checkFeatureActive("apphosting", ctx.projectId, { config: ctx.config }), + apptesting: isAppTestingAvailable, database: (ctx: McpContext): Promise => checkFeatureActive("database", ctx.projectId, { config: ctx.config }), }; From 8a95353f98ecc9c3092d211e776ac4e808c3b6f4 Mon Sep 17 00:00:00 2001 From: tagboola Date: Tue, 28 Oct 2025 10:18:08 -0400 Subject: [PATCH 10/19] Add get devices tool (#9387) --- src/mcp/tools/apptesting/devices.ts | 26 ++++++++++++++++++++++++++ src/mcp/tools/apptesting/index.ts | 3 ++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 src/mcp/tools/apptesting/devices.ts diff --git a/src/mcp/tools/apptesting/devices.ts b/src/mcp/tools/apptesting/devices.ts new file mode 100644 index 00000000000..d3b2f67f6a4 --- /dev/null +++ b/src/mcp/tools/apptesting/devices.ts @@ -0,0 +1,26 @@ +import { z } from "zod"; +import { tool } from "../../tool"; +import { google } from "googleapis"; +import { toContent } from "../../util"; +import { getAccessToken } from "../../../apiv2"; + +export const get_devices = tool( + "apptesting", + { + name: "get_devices", + description: + "Get available devices that can be used for automated tests using the app testing agent", + inputSchema: z.object({ + type: z.enum(["ANDROID"]).describe("The type of device"), + }), + }, + async ({ type }) => { + const testing = google.testing("v1"); + return toContent( + await testing.testEnvironmentCatalog.get({ + oauth_token: await getAccessToken(), + environmentType: type, + }), + ); + }, +); diff --git a/src/mcp/tools/apptesting/index.ts b/src/mcp/tools/apptesting/index.ts index 22592e537ce..763fc55e128 100644 --- a/src/mcp/tools/apptesting/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,4 +1,5 @@ import type { ServerTool } from "../../tool"; +import { get_devices } from "./devices"; import { check_test, run_tests } from "./tests"; -export const apptestingTools: ServerTool[] = [run_tests, check_test]; +export const apptestingTools: ServerTool[] = [run_tests, check_test, get_devices]; From 7356c416ea5deeab1e27b0e318c4bce43a53fa8f Mon Sep 17 00:00:00 2001 From: tagboola Date: Wed, 29 Oct 2025 15:27:32 -0400 Subject: [PATCH 11/19] Display link to results in the Firebase Console (#9406) --- src/mcp/prompts/apptesting/run_test.ts | 9 ++++++++- src/mcp/tools/apptesting/tests.ts | 10 ++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 536bd6122b3..cb5800c5ebb 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -106,10 +106,17 @@ Here are a list of prerequisite steps that must be completed before running a te ## Run Test - Use the apptesting_run_test tool to run an automated test with the following as input: + Use the \`apptesting_run_test\` tool to run an automated test with the following as input: * The generated test case that as been confirmed by the user * An APK. If there is no APK present, build the app to produce one. Make sure to build the variant of the app with the same bundle ID as the Firebase app. + * Once the test has started, provide the developer a link to see the results of the test in the Firebase Console. + You should already know the value of \`appId\' and \`projectId\` from earlier (if you only know \`projectNumber\', + use the \`firebase_get_project\` tool to get \`projectId\`). The \`apptesting_run_test\` tool returns a response + with field \`name\` in the form projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. + Extract the values for \'releaseId\' and \`releaseTestId\` and use provide a link to the results in the Firebase + Console in the format: + \`https://console.firebase.google.com/u/0/project/{projectId}/apptesting/app/{appId}/releases/{releaseId}/tests/{releaseTestId}\`. `.trim(), }, }, diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index fd298adbadd..eda41caef24 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -50,9 +50,11 @@ export const run_tests = tool( orientation: "portrait", }, ]), - testCase: z - .array(AIStepSchema) - .describe("Test case containing the steps that are run during its execution."), + testCase: z.object({ + steps: z + .array(AIStepSchema) + .describe("Test case containing the steps that are run during its execution."), + }), }), annotations: { title: "Run a Remote Test", @@ -62,7 +64,7 @@ export const run_tests = tool( async ({ appId, releaseBinaryFile, testDevices, testCase }) => { const client = new AppDistributionClient(); const releaeName = await upload(client, toAppName(appId), new Distribution(releaseBinaryFile)); - return toContent(await client.createReleaseTest(releaeName, testDevices, { steps: testCase })); + return toContent(await client.createReleaseTest(releaeName, testDevices, testCase)); }, ); From 7f948283ceda7de2403b33442aa9a315db5c21c4 Mon Sep 17 00:00:00 2001 From: Tunde Agboola Date: Thu, 30 Oct 2025 10:01:51 -0400 Subject: [PATCH 12/19] Place app testing tools behind an experiment --- src/mcp/tools/apptesting/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mcp/tools/apptesting/index.ts b/src/mcp/tools/apptesting/index.ts index 763fc55e128..b27084ff095 100644 --- a/src/mcp/tools/apptesting/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,5 +1,10 @@ +import { isEnabled } from "../../../experiments"; import type { ServerTool } from "../../tool"; import { get_devices } from "./devices"; import { check_test, run_tests } from "./tests"; -export const apptestingTools: ServerTool[] = [run_tests, check_test, get_devices]; +export const apptestingTools: ServerTool[] = []; + +if (isEnabled("mcpalpha")) { + apptestingTools.push(...[run_tests, check_test, get_devices]); +} From cb9c42b5b2c2a1b11acdd691f5e2b756725b6156 Mon Sep 17 00:00:00 2001 From: Tunde Agboola Date: Thu, 30 Oct 2025 10:16:51 -0400 Subject: [PATCH 13/19] Address GCA comments --- src/appdistribution/distribution.ts | 7 ++++--- src/mcp/tools/apptesting/tests.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index 883f225a346..b81a7e25848 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -141,23 +141,24 @@ export async function awaitTestResults( } } for (const execution of releaseTest.deviceExecutions) { + const device = deviceToString(execution.device); switch (execution.state) { case "PASSED": case "IN_PROGRESS": continue; case "FAILED": throw new FirebaseError( - `Automated test failed for ${deviceToString(execution.device)}: ${execution.failedReason}`, + `Automated test failed for ${device}: ${execution.failedReason}`, { exit: 1 }, ); case "INCONCLUSIVE": throw new FirebaseError( - `Automated test inconclusive for ${deviceToString(execution.device)}: ${execution.inconclusiveReason}`, + `Automated test inconclusive for ${device}: ${execution.inconclusiveReason}`, { exit: 1 }, ); default: throw new FirebaseError( - `Unsupported automated test state for ${deviceToString(execution.device)}: ${execution.state}`, + `Unsupported automated test state for ${device}: ${execution.state}`, { exit: 1 }, ); } diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index eda41caef24..1cab84cee80 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -63,8 +63,8 @@ export const run_tests = tool( }, async ({ appId, releaseBinaryFile, testDevices, testCase }) => { const client = new AppDistributionClient(); - const releaeName = await upload(client, toAppName(appId), new Distribution(releaseBinaryFile)); - return toContent(await client.createReleaseTest(releaeName, testDevices, testCase)); + const releaseName = await upload(client, toAppName(appId), new Distribution(releaseBinaryFile)); + return toContent(await client.createReleaseTest(releaseName, testDevices, testCase)); }, ); From 81a2734367984b70c13bd7dfabdb7cec78d969e3 Mon Sep 17 00:00:00 2001 From: Tunde Agboola Date: Fri, 31 Oct 2025 14:51:25 -0400 Subject: [PATCH 14/19] Explicitly set default devices --- src/mcp/tools/apptesting/tests.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 1cab84cee80..4b9d7c161ba 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -34,6 +34,15 @@ const AIStepSchema = z }) .describe("Step within a test case; run during the execution of the test."); +const defaultDevices = [ + { + model: "MediumPhone.arm", + version: "30", + locale: "en_US", + orientation: "portrait", + }, +]; + export const run_tests = tool( "apptesting", { @@ -42,14 +51,7 @@ export const run_tests = tool( inputSchema: z.object({ appId: ApplicationIdSchema, releaseBinaryFile: z.string().describe("Path to the binary release (APK)."), - testDevices: z.array(TestDeviceSchema).default([ - { - model: "MediumPhone.arm", - version: "30", - locale: "en_US", - orientation: "portrait", - }, - ]), + testDevices: z.array(TestDeviceSchema).default(defaultDevices), testCase: z.object({ steps: z .array(AIStepSchema) @@ -62,9 +64,11 @@ export const run_tests = tool( }, }, async ({ appId, releaseBinaryFile, testDevices, testCase }) => { + // For some reason, testDevices can still be + const devices = testDevices || defaultDevices; const client = new AppDistributionClient(); const releaseName = await upload(client, toAppName(appId), new Distribution(releaseBinaryFile)); - return toContent(await client.createReleaseTest(releaseName, testDevices, testCase)); + return toContent(await client.createReleaseTest(releaseName, devices, testCase)); }, ); From 0bc82085c895dfff9935e1c84627198bc6126d0f Mon Sep 17 00:00:00 2001 From: Tunde Agboola Date: Fri, 7 Nov 2025 11:45:35 -0500 Subject: [PATCH 15/19] Address PR comments --- src/appdistribution/distribution.ts | 2 ++ src/mcp/prompts/apptesting/run_test.ts | 25 +++++++++------ src/mcp/tools/apptesting/devices.ts | 26 --------------- src/mcp/tools/apptesting/index.ts | 5 ++- src/mcp/tools/apptesting/tests.ts | 44 ++++++++++++++++++++++---- 5 files changed, 56 insertions(+), 46 deletions(-) delete mode 100644 src/mcp/tools/apptesting/devices.ts diff --git a/src/appdistribution/distribution.ts b/src/appdistribution/distribution.ts index b81a7e25848..4a006103080 100644 --- a/src/appdistribution/distribution.ts +++ b/src/appdistribution/distribution.ts @@ -15,6 +15,7 @@ export enum DistributionFileType { AAB = "aab", } +/** Upload a distribution */ export async function upload( requests: AppDistributionClient, appName: string, @@ -119,6 +120,7 @@ export class Distribution { } } +/** Wait for release tests to complete */ export async function awaitTestResults( releaseTests: ReleaseTest[], requests: AppDistributionClient, diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index cb5800c5ebb..11f3a457c18 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -37,15 +37,12 @@ Here are a list of prerequisite steps that must be completed before running a te 1. **Make sure this is an Android app**. The App Testing agent only works with Android apps. If this is not an Android app, instruct the user that this command can't be used with this app. - 2. **Make sure the user is logged in. No App Testing tools will work if the user is not logged in.** a. Use the \`firebase_get_environment\` tool to verify that the user is logged in. b. If the Firebase 'Active user' is set to , instruct the user to run \`firebase login\` before continuing. Ignore other fields that are set to . We are just making sure the user is logged in. - 3. **Get the Firebase app ID.** - The \`firebase_get_environment\` tool should return a list of detected app IDs, where the app ID contains four colon (":") delimited parts: a version number (typically "1"), a project number, a platform type ("android", "ios", or "web"). Ask the user confirm if there is only @@ -110,13 +107,21 @@ Here are a list of prerequisite steps that must be completed before running a te * The generated test case that as been confirmed by the user * An APK. If there is no APK present, build the app to produce one. Make sure to build the variant of the app with the same bundle ID as the Firebase app. - * Once the test has started, provide the developer a link to see the results of the test in the Firebase Console. - You should already know the value of \`appId\' and \`projectId\` from earlier (if you only know \`projectNumber\', - use the \`firebase_get_project\` tool to get \`projectId\`). The \`apptesting_run_test\` tool returns a response - with field \`name\` in the form projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. - Extract the values for \'releaseId\' and \`releaseTestId\` and use provide a link to the results in the Firebase - Console in the format: - \`https://console.firebase.google.com/u/0/project/{projectId}/apptesting/app/{appId}/releases/{releaseId}/tests/{releaseTestId}\`. + * The devices to test on. If the user doesn't specify any devices in the test description, you can leave this + blank and the test will run on a the default virtual device. If the user does specify a device, + Use the \`apptesting_check_status\` tool with \`getAvailableDevices\` set to true to get a list of available + devices. + + Once the test has started, provide the developer a link to see the results of the test in the Firebase Console. + You should already know the value of \`appId\' and \`projectId\` from earlier (if you only know \`projectNumber\', + use the \`firebase_get_project\` tool to get \`projectId\`). The \`apptesting_run_test\` tool returns a response + with field \`name\` in the form projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. + Extract the values for \'releaseId\' and \`releaseTestId\` and use provide a link to the results in the Firebase + Console in the format: + \`https://console.firebase.google.com/u/0/project/{projectId}/apptesting/app/{appId}/releases/{releaseId}/tests/{releaseTestId}\`. + + You can check the status of the test using the \`apptesting_check_status\` tool with \`release_test_name\' set to + the name of the release test returned by the \`run_test\` tool. `.trim(), }, }, diff --git a/src/mcp/tools/apptesting/devices.ts b/src/mcp/tools/apptesting/devices.ts deleted file mode 100644 index d3b2f67f6a4..00000000000 --- a/src/mcp/tools/apptesting/devices.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { z } from "zod"; -import { tool } from "../../tool"; -import { google } from "googleapis"; -import { toContent } from "../../util"; -import { getAccessToken } from "../../../apiv2"; - -export const get_devices = tool( - "apptesting", - { - name: "get_devices", - description: - "Get available devices that can be used for automated tests using the app testing agent", - inputSchema: z.object({ - type: z.enum(["ANDROID"]).describe("The type of device"), - }), - }, - async ({ type }) => { - const testing = google.testing("v1"); - return toContent( - await testing.testEnvironmentCatalog.get({ - oauth_token: await getAccessToken(), - environmentType: type, - }), - ); - }, -); diff --git a/src/mcp/tools/apptesting/index.ts b/src/mcp/tools/apptesting/index.ts index b27084ff095..e2fb94dc5b6 100644 --- a/src/mcp/tools/apptesting/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,10 +1,9 @@ import { isEnabled } from "../../../experiments"; import type { ServerTool } from "../../tool"; -import { get_devices } from "./devices"; -import { check_test, run_tests } from "./tests"; +import { check_status, run_tests } from "./tests"; export const apptestingTools: ServerTool[] = []; if (isEnabled("mcpalpha")) { - apptestingTools.push(...[run_tests, check_test, get_devices]); + apptestingTools.push(...[run_tests, check_status]); } diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 4b9d7c161ba..57548b2a0e7 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -6,6 +6,8 @@ import { tool } from "../../tool"; import { toContent } from "../../util"; import { toAppName } from "../../../appdistribution/options-parser-util"; import { AppDistributionClient } from "../../../appdistribution/client"; +import { google } from "googleapis"; +import { getAccessToken } from "../../../apiv2"; const TestDeviceSchema = z .object({ @@ -72,21 +74,49 @@ export const run_tests = tool( }, ); -export const check_test = tool( +export const check_status = tool( "apptesting", { - name: "check_test", - description: "Check the status of a remote test.", + name: "check_status", + description: + "Check the status of an apptesting release test and/or get available devices that can be used for automated tests ", inputSchema: z.object({ - name: z.string().describe("The name of the release test returned by the run_test tool."), + release_test_name: z + .string() + .optional() + .describe( + "The name of the release test returned by the run_test tool. If set, the tool will fetch the release test", + ), + getAvailableDevices: z + .boolean() + .optional() + .describe( + "If set to true, the tool will get the available devices that can be used for automated tests using the app testing agent", + ), }), annotations: { title: "Check Remote Test", readOnlyHint: true, }, }, - async ({ name }) => { - const client = new AppDistributionClient(); - return toContent(await client.getReleaseTest(name)); + async ({ release_test_name, getAvailableDevices }) => { + let devices = undefined; + let releaseTest = undefined; + if (release_test_name) { + const client = new AppDistributionClient(); + releaseTest = await client.getReleaseTest(release_test_name); + } + if (getAvailableDevices) { + const testing = google.testing("v1"); + devices = await testing.testEnvironmentCatalog.get({ + oauth_token: await getAccessToken(), + environmentType: "ANDROID", + }); + } + + return toContent({ + devices, + releaseTest, + }); }, ); From 9877ee5d6cf19e58eeb2494486c22ab914d30a4f Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Fri, 7 Nov 2025 12:49:38 -0500 Subject: [PATCH 16/19] Fix the status URL. (#9438) Co-authored-by: Jamie Rothfeder --- src/mcp/prompts/apptesting/run_test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mcp/prompts/apptesting/run_test.ts b/src/mcp/prompts/apptesting/run_test.ts index 11f3a457c18..767da9e2cb4 100644 --- a/src/mcp/prompts/apptesting/run_test.ts +++ b/src/mcp/prompts/apptesting/run_test.ts @@ -114,11 +114,11 @@ Here are a list of prerequisite steps that must be completed before running a te Once the test has started, provide the developer a link to see the results of the test in the Firebase Console. You should already know the value of \`appId\' and \`projectId\` from earlier (if you only know \`projectNumber\', - use the \`firebase_get_project\` tool to get \`projectId\`). The \`apptesting_run_test\` tool returns a response - with field \`name\` in the form projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. - Extract the values for \'releaseId\' and \`releaseTestId\` and use provide a link to the results in the Firebase - Console in the format: - \`https://console.firebase.google.com/u/0/project/{projectId}/apptesting/app/{appId}/releases/{releaseId}/tests/{releaseTestId}\`. + use the \`firebase_get_project\` tool to get \`projectId\`). \`packageName\` is the package name of the app we tested. + The \`apptesting_run_test\` tool returns a response with field \`name\` in the form + projects/{projectNumber}/apps/{appId}/releases/{releaseId}/tests/{releaseTestId}. Extract the values for \'releaseId\' + and \`releaseTestId\` and use provide a link to the results in the Firebase Console in the format: + \`https://console.firebase.google.com/u/0/project/{projectId}/apptesting/app/android:{packageName}/releases/{releaseId}/tests/{releaseTestId}\`. You can check the status of the test using the \`apptesting_check_status\` tool with \`release_test_name\' set to the name of the release test returned by the \`run_test\` tool. From 1c4631d36b3632fa33484329a072ebf97302f503 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Fri, 7 Nov 2025 16:47:15 -0500 Subject: [PATCH 17/19] Add a tool to export tests to a file. --- src/appdistribution/yaml_helper.ts | 4 +++- src/mcp/tools/apptesting/index.ts | 4 ++-- src/mcp/tools/apptesting/tests.ts | 28 ++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 466467dafdc..5b11e1ae269 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -31,7 +31,9 @@ function extractIdFromResourceName(name: string): string { function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { return testCases.map((testCase) => ({ displayName: testCase.displayName, - id: extractIdFromResourceName(testCase.name!), // resource name is retured by server + ...(testCase.name && { + id: extractIdFromResourceName(testCase.name!), // resource name is retured by server + }), ...(testCase.prerequisiteTestCase && { prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase), }), diff --git a/src/mcp/tools/apptesting/index.ts b/src/mcp/tools/apptesting/index.ts index e2fb94dc5b6..12d277bc987 100644 --- a/src/mcp/tools/apptesting/index.ts +++ b/src/mcp/tools/apptesting/index.ts @@ -1,9 +1,9 @@ import { isEnabled } from "../../../experiments"; import type { ServerTool } from "../../tool"; -import { check_status, run_tests } from "./tests"; +import { check_status, run_tests, testcase_export } from "./tests"; export const apptestingTools: ServerTool[] = []; if (isEnabled("mcpalpha")) { - apptestingTools.push(...[run_tests, check_status]); + apptestingTools.push(...[run_tests, check_status, testcase_export]); } diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index 57548b2a0e7..3e714617e8a 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -1,6 +1,8 @@ import { z } from "zod"; +import * as fs from "fs-extra"; import { ApplicationIdSchema } from "../../../crashlytics/filters"; import { upload, Distribution } from "../../../appdistribution/distribution"; +import { toYaml } from "../../../appdistribution/yaml_helper"; import { tool } from "../../tool"; import { toContent } from "../../util"; @@ -74,6 +76,32 @@ export const run_tests = tool( }, ); +export const testcase_export = tool( + "apptesting", + { + name: "testcase_export", + description: "Use this to export a testcases to a file.", + inputSchema: z.object({ + outputFile: z.string().describe("The path to the file."), + testCases: z.array( + z.object({ + displayName: z.string(), + aiInstructions: z.object({ + steps: z.array(AIStepSchema), + }), + }), + ), + }), + annotations: { + title: "Export testcases to a file.", + readOnlyHint: false, + }, + }, + async ({ outputFile, testCases }) => { + return toContent(fs.writeFileSync(outputFile, toYaml(testCases), "utf8")); + }, +); + export const check_status = tool( "apptesting", { From a1de3cd5383d1dd86ba88ca1f705af85d869043c Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Mon, 10 Nov 2025 13:21:58 -0500 Subject: [PATCH 18/19] Remove merge conflict markers. --- src/mcp/tools/apptesting/tests.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/mcp/tools/apptesting/tests.ts b/src/mcp/tools/apptesting/tests.ts index b551120df27..8ac20675c09 100644 --- a/src/mcp/tools/apptesting/tests.ts +++ b/src/mcp/tools/apptesting/tests.ts @@ -1,13 +1,8 @@ import { z } from "zod"; -<<<<<<< HEAD import * as fs from "fs-extra"; import { ApplicationIdSchema } from "../../../crashlytics/filters"; import { upload, Distribution } from "../../../appdistribution/distribution"; import { toYaml } from "../../../appdistribution/yaml_helper"; -======= -import { ApplicationIdSchema } from "../../../crashlytics/filters"; -import { upload, Distribution } from "../../../appdistribution/distribution"; ->>>>>>> master import { tool } from "../../tool"; import { toContent } from "../../util"; @@ -81,12 +76,11 @@ export const run_tests = tool( }, ); -<<<<<<< HEAD export const testcase_export = tool( "apptesting", { name: "testcase_export", - description: "Use this to export a testcases to a file.", + description: "Use this to export a testcase to a file.", inputSchema: z.object({ outputFile: z.string().describe("The path to the file."), testCases: z.array( @@ -108,8 +102,6 @@ export const testcase_export = tool( }, ); -======= ->>>>>>> master export const check_status = tool( "apptesting", { From 6d6b1a999c12879c46ade8df3a7fa02f1434b289 Mon Sep 17 00:00:00 2001 From: Jamie Rothfeder Date: Mon, 10 Nov 2025 13:26:36 -0500 Subject: [PATCH 19/19] Simplify conversion of testcases to yaml. --- src/appdistribution/yaml_helper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/appdistribution/yaml_helper.ts b/src/appdistribution/yaml_helper.ts index 5b11e1ae269..eebaf432d91 100644 --- a/src/appdistribution/yaml_helper.ts +++ b/src/appdistribution/yaml_helper.ts @@ -32,7 +32,7 @@ function toYamlTestCases(testCases: TestCase[]): YamlTestCase[] { return testCases.map((testCase) => ({ displayName: testCase.displayName, ...(testCase.name && { - id: extractIdFromResourceName(testCase.name!), // resource name is retured by server + id: extractIdFromResourceName(testCase.name), }), ...(testCase.prerequisiteTestCase && { prerequisiteTestCaseId: extractIdFromResourceName(testCase.prerequisiteTestCase),