From 96559553d4cb409ab84a857811bef6af78ca8d6f Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 15 Apr 2025 15:00:31 -0400 Subject: [PATCH 1/2] [Vertex AI] Temporarily disable proxied Dev API integration tests --- .../CountTokensIntegrationTests.swift | 20 ++++++------------- .../GenerateContentIntegrationTests.swift | 13 ++++-------- .../Tests/Utilities/InstanceConfig.swift | 10 +++++++++- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift index ee416f76329..96e3b5da5c0 100644 --- a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift +++ b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift @@ -69,14 +69,10 @@ struct CountTokensIntegrationTests { #expect(promptTokensDetails.tokenCount == response.totalTokens) } - @Test(arguments: [ - InstanceConfig.vertexV1, - InstanceConfig.vertexV1Staging, - InstanceConfig.vertexV1Beta, - InstanceConfig.vertexV1BetaStaging, + @Test( /* System instructions are not supported on the v1 Developer API. */ - InstanceConfig.developerV1Beta, - ]) + arguments: InstanceConfig.allConfigsExceptDeveloperV1 + ) func countTokens_text_systemInstruction(_ config: InstanceConfig) async throws { let model = VertexAI.componentInstance(config).generativeModel( modelName: ModelNames.gemini2Flash, @@ -122,14 +118,10 @@ struct CountTokensIntegrationTests { ) } - @Test(arguments: [ - InstanceConfig.vertexV1, - InstanceConfig.vertexV1Staging, - InstanceConfig.vertexV1Beta, - InstanceConfig.vertexV1BetaStaging, + @Test( /* System instructions are not supported on the v1 Developer API. */ - InstanceConfig.developerV1Beta, - ]) + arguments: InstanceConfig.allConfigsExceptDeveloperV1 + ) func countTokens_jsonSchema(_ config: InstanceConfig) async throws { let model = VertexAI.componentInstance(config).generativeModel( modelName: ModelNames.gemini2Flash, diff --git a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift index f1650f26471..0f06131ce8a 100644 --- a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift +++ b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/GenerateContentIntegrationTests.swift @@ -77,14 +77,8 @@ struct GenerateContentIntegrationTests { @Test( "Generate an enum and provide a system instruction", - arguments: [ - InstanceConfig.vertexV1, - InstanceConfig.vertexV1Staging, - InstanceConfig.vertexV1Beta, - InstanceConfig.vertexV1BetaStaging, - /* System instructions are not supported on the v1 Developer API. */ - InstanceConfig.developerV1Beta, - ] + /* System instructions are not supported on the v1 Developer API. */ + arguments: InstanceConfig.allConfigsExceptDeveloperV1 ) func generateContentEnum(_ config: InstanceConfig) async throws { let model = VertexAI.componentInstance(config).generativeModel( @@ -122,7 +116,8 @@ struct GenerateContentIntegrationTests { @Test(arguments: [ InstanceConfig.vertexV1Beta, - InstanceConfig.developerV1Beta, + // Temporarily disabled due to Firebase Proxy issues. + // InstanceConfig.developerV1Beta, ]) func generateImage(_ config: InstanceConfig) async throws { let generationConfig = GenerationConfig( diff --git a/FirebaseVertexAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift b/FirebaseVertexAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift index bb4a0766eac..2bfd135d562 100644 --- a/FirebaseVertexAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift +++ b/FirebaseVertexAI/Tests/TestApp/Tests/Utilities/InstanceConfig.swift @@ -35,6 +35,9 @@ struct InstanceConfig { static let developerV1Beta = InstanceConfig( apiConfig: APIConfig(service: .developer(endpoint: .firebaseVertexAIProd), version: .v1beta) ) + static let developerV1BetaStaging = InstanceConfig( + apiConfig: APIConfig(service: .developer(endpoint: .firebaseVertexAIStaging), version: .v1beta) + ) static let developerV1Spark = InstanceConfig( appName: FirebaseAppNames.spark, apiConfig: APIConfig(service: .developer(endpoint: .generativeLanguage), version: .v1) @@ -48,7 +51,9 @@ struct InstanceConfig { vertexV1Staging, vertexV1Beta, vertexV1BetaStaging, - developerV1Beta, + // Temporarily disabled due to Firebase Proxy issues: + // developerV1Beta, + // developerV1BetaStaging, developerV1Spark, developerV1BetaSpark, ] @@ -58,6 +63,9 @@ struct InstanceConfig { vertexV1Staging, vertexV1Beta, vertexV1BetaStaging, + // Temporarily disabled due to Firebase Proxy issues: + // developerV1Beta, + // developerV1BetaStaging, developerV1BetaSpark, ] From a5eea9b43b31e820f402d3f817af648ee2e64522 Mon Sep 17 00:00:00 2001 From: Andrew Heard Date: Tue, 15 Apr 2025 16:37:59 -0400 Subject: [PATCH 2/2] Update `totalBillableCharacters` count to match new backend counts --- .../TestApp/Tests/Integration/CountTokensIntegrationTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift index 96e3b5da5c0..a0abd8cc00d 100644 --- a/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift +++ b/FirebaseVertexAI/Tests/TestApp/Tests/Integration/CountTokensIntegrationTests.swift @@ -143,7 +143,7 @@ struct CountTokensIntegrationTests { switch config.apiConfig.service { case .vertexAI: #expect(response.totalTokens == 65) - #expect(response.totalBillableCharacters == 165) + #expect(response.totalBillableCharacters == 170) case .developer: // The Developer API erroneously ignores the `responseSchema` when counting tokens, resulting // in a lower total count than Vertex AI.