Merge branch 'dev-v3' into hb-us-23028

Author: Harsh-Microsoft

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ on:
     - cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT
   workflow_dispatch: #Allow manual triggering
 env:
-  GPT_MIN_CAPACITY: 150
+  GPT_MIN_CAPACITY: 1
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
 jobs:
@@ -36,7 +36,7 @@ jobs:
           export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
           export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
           export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          export GPT_MIN_CAPACITY="150"
+          export GPT_MIN_CAPACITY="1"
           export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
 
           chmod +x infra/scripts/checkquota.sh
@@ -140,7 +140,7 @@ jobs:
               backendContainerImageTag="${IMAGE_TAG}" \
               frontendContainerImageTag="${IMAGE_TAG}" \
               azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \
-              gptModelCapacity=150 \
+              gptModelCapacity=1 \
               createdBy="Pipeline" \
               tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}" \
             --output json
@@ -181,20 +181,9 @@ jobs:
             echo "SUCCESS=false" >> $GITHUB_OUTPUT
           fi
 
-  e2e-test:
-    needs: deploy
-    if: needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
-    uses: ./.github/workflows/test-automation.yml
-    with:
-      MACAE_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }}
-      MACAE_URL_API: ${{ needs.deploy.outputs.MACAE_URL_API }}
-      MACAE_RG: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
-      MACAE_CONTAINER_APP: ${{ needs.deploy.outputs.CONTAINER_APP }}
-    secrets: inherit
-
   cleanup-deployment:
     if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
-    needs: [deploy, e2e-test]
+    needs: [deploy]
     runs-on: ubuntu-latest
     env:
       RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}

.github/workflows/test.yml

@@ -61,7 +61,9 @@ jobs:
             --ignore=src/backend/tests/test_config.py \
             --ignore=src/tests/agents/test_human_approval_manager.py \
             --ignore=src/backend/tests/test_team_specific_methods.py \
-            --ignore=src/backend/tests/models/test_messages.py
+            --ignore=src/backend/tests/models/test_messages.py \
+            --ignore=src/backend/tests/test_otlp_tracing.py \
+            --ignore=src/backend/tests/auth/test_auth_utils.py
 
       # - name: Run tests with coverage
       #   if: env.skip_tests == 'false'

src/backend/app_kernel.py

@@ -1,23 +1,50 @@
 # app_kernel.py
-import asyncio
 import logging
-import os
-# Azure monitoring
-import re
-import uuid
-from typing import Dict, List, Optional
+
+from contextlib import asynccontextmanager
+
 
 from azure.monitor.opentelemetry import configure_azure_monitor
 from common.config.app_config import config
 from common.models.messages_kernel import UserLanguage
+
 # FastAPI imports
-from fastapi import FastAPI, Query, Request
+from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
+
 # Local imports
 from middleware.health_check import HealthCheckMiddleware
 from v3.api.router import app_v3
+
+# Azure monitoring
+
 # Semantic Kernel imports
-from v3.orchestration.orchestration_manager import OrchestrationManager
+from v3.config.agent_registry import agent_registry
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage FastAPI application lifecycle - startup and shutdown."""
+    logger = logging.getLogger(__name__)
+
+    # Startup
+    logger.info("🚀 Starting MACAE application...")
+    yield
+
+    # Shutdown
+    logger.info("🛑 Shutting down MACAE application...")
+    try:
+        # Clean up all agents from Azure AI Foundry when container stops
+        await agent_registry.cleanup_all_agents()
+        logger.info("✅ Agent cleanup completed successfully")
+
+    except ImportError as ie:
+        logger.error(f"❌ Could not import agent_registry: {ie}")
+    except Exception as e:
+        logger.error(f"❌ Error during shutdown cleanup: {e}")
+
+    logger.info("👋 MACAE application shutdown complete")
+
 
 # Check if the Application Insights Instrumentation Key is set in the environment variables
 connection_string = config.APPLICATIONINSIGHTS_CONNECTION_STRING
@@ -48,7 +75,7 @@
 )
 
 # Initialize the FastAPI app
-app = FastAPI()
+app = FastAPI(lifespan=lifespan)
 
 frontend_url = config.FRONTEND_SITE_NAME
 
@@ -104,4 +131,11 @@ async def user_browser_language_endpoint(user_language: UserLanguage, request: R
 if __name__ == "__main__":
     import uvicorn
 
-    uvicorn.run("app_kernel:app", host="127.0.0.1", port=8000, reload=True, log_level="info", access_log=False)
+    uvicorn.run(
+        "app_kernel:app",
+        host="127.0.0.1",
+        port=8000,
+        reload=True,
+        log_level="info",
+        access_log=False,
+    )

src/backend/common/config/app_config.py

@@ -76,15 +76,23 @@ def __init__(self):
         # Optional MCP server endpoint (for local MCP server or remote)
         # Example: http://127.0.0.1:8000/mcp
         self.MCP_SERVER_ENDPOINT = self._get_optional("MCP_SERVER_ENDPOINT")
-        self.MCP_SERVER_NAME = self._get_optional("MCP_SERVER_NAME", "MCPGreetingServer")
-        self.MCP_SERVER_DESCRIPTION = self._get_optional("MCP_SERVER_DESCRIPTION", "MCP server with greeting and planning tools")
+        self.MCP_SERVER_NAME = self._get_optional(
+            "MCP_SERVER_NAME", "MCPGreetingServer"
+        )
+        self.MCP_SERVER_DESCRIPTION = self._get_optional(
+            "MCP_SERVER_DESCRIPTION", "MCP server with greeting and planning tools"
+        )
         self.TENANT_ID = self._get_optional("AZURE_TENANT_ID")
         self.CLIENT_ID = self._get_optional("AZURE_CLIENT_ID")
-        self.AZURE_AI_SEARCH_CONNECTION_NAME = self._get_optional("AZURE_AI_SEARCH_CONNECTION_NAME")
-        self.AZURE_AI_SEARCH_INDEX_NAME = self._get_optional("AZURE_AI_SEARCH_INDEX_NAME")
+        self.AZURE_AI_SEARCH_CONNECTION_NAME = self._get_optional(
+            "AZURE_AI_SEARCH_CONNECTION_NAME"
+        )
+        self.AZURE_AI_SEARCH_INDEX_NAME = self._get_optional(
+            "AZURE_AI_SEARCH_INDEX_NAME"
+        )
         self.AZURE_AI_SEARCH_ENDPOINT = self._get_optional("AZURE_AI_SEARCH_ENDPOINT")
         self.AZURE_AI_SEARCH_API_KEY = self._get_optional("AZURE_AI_SEARCH_API_KEY")
-        # self.BING_CONNECTION_NAME = self._get_optional("BING_CONNECTION_NAME")  
+        # self.BING_CONNECTION_NAME = self._get_optional("BING_CONNECTION_NAME")
 
         test_team_json = self._get_optional("TEST_TEAM_JSON")
 
@@ -117,7 +125,7 @@ def get_azure_credential(self, client_id=None):
             )  # CodeQL [SM05139] Okay use of DefaultAzureCredential as it is only used in development
         else:
             return ManagedIdentityCredential(client_id=client_id)
-        
+
     def get_azure_credentials(self):
         """Retrieve Azure credentials, either from environment variables or managed identity."""
         if self._azure_credentials is None:
@@ -192,7 +200,8 @@ def get_cosmos_database_client(self):
         try:
             if self._cosmos_client is None:
                 self._cosmos_client = CosmosClient(
-                    self.COSMOSDB_ENDPOINT, credential=self.get_azure_credential(self.AZURE_CLIENT_ID)
+                    self.COSMOSDB_ENDPOINT,
+                    credential=self.get_azure_credential(self.AZURE_CLIENT_ID),
                 )
 
             if self._cosmos_database is None:

src/backend/common/database/cosmosdb.py

@@ -1,37 +1,24 @@
 """CosmosDB implementation of the database interface."""
 
-import json
-import logging
-import uuid
-
 import datetime
+import logging
 from typing import Any, Dict, List, Optional, Type
 
-from azure.cosmos import PartitionKey, exceptions
+import v3.models.messages as messages
 from azure.cosmos.aio import CosmosClient
 from azure.cosmos.aio._database import DatabaseProxy
-from azure.cosmos.exceptions import CosmosResourceExistsError
-import v3.models.messages as messages
 
-from common.models.messages_kernel import (
-    AgentMessage,
-    Plan,
-    Step,
-    TeamConfiguration,
-)
-from common.utils.utils_date import DateTimeEncoder
-
-from .database_base import DatabaseBase
 from ..models.messages_kernel import (
+    AgentMessage,
     AgentMessageData,
     BaseDataModel,
+    DataType,
     Plan,
     Step,
-    AgentMessage,
     TeamConfiguration,
-    DataType,
     UserCurrentTeam,
 )
+from .database_base import DatabaseBase
 
 
 class CosmosDBClient(DatabaseBase):
@@ -189,7 +176,6 @@ async def delete_item(self, item_id: str, partition_key: str) -> None:
             self.logger.error("Failed to delete item from CosmosDB: %s", str(e))
             raise
 
-
     # Plan Operations
     async def add_plan(self, plan: Plan) -> None:
         """Add a plan to CosmosDB."""
@@ -199,7 +185,6 @@ async def update_plan(self, plan: Plan) -> None:
         """Update a plan in CosmosDB."""
         await self.update_item(plan)
 
-
     async def get_plan_by_plan_id(self, plan_id: str) -> Optional[Plan]:
         """Retrieve a plan by plan_id."""
         query = "SELECT * FROM c WHERE c.id=@plan_id AND c.data_type=@data_type"
@@ -234,8 +219,9 @@ async def get_all_plans_by_team_id(self, team_id: str) -> List[Plan]:
         ]
         return await self.query_items(query, parameters, Plan)
 
-
-    async def get_all_plans_by_team_id_status(self, user_id: str,team_id: str, status: str) -> List[Plan]:
+    async def get_all_plans_by_team_id_status(
+        self, user_id: str, team_id: str, status: str
+    ) -> List[Plan]:
         """Retrieve all plans for a specific team."""
         query = "SELECT * FROM c WHERE c.team_id=@team_id AND c.data_type=@data_type and c.user_id=@user_id and c.overall_status=@status ORDER BY c._ts DESC"
         parameters = [
@@ -245,6 +231,7 @@ async def get_all_plans_by_team_id_status(self, user_id: str,team_id: str, statu
             {"name": "@status", "value": status},
         ]
         return await self.query_items(query, parameters, Plan)
+
     # Step Operations
     async def add_step(self, step: Step) -> None:
         """Add a step to CosmosDB."""
@@ -414,8 +401,6 @@ async def get_current_team(self, user_id: str) -> Optional[UserCurrentTeam]:
         teams = await self.query_items(query, parameters, UserCurrentTeam)
         return teams[0] if teams else None
 
-
-
     async def delete_current_team(self, user_id: str) -> bool:
         """Delete the current team for a user."""
         query = "SELECT c.id, c.session_id FROM c WHERE c.user_id=@user_id AND c.data_type=@data_type"
@@ -429,9 +414,13 @@ async def delete_current_team(self, user_id: str) -> bool:
         if items:
             async for doc in items:
                 try:
-                    await self.container.delete_item(doc["id"], partition_key=doc["session_id"])
+                    await self.container.delete_item(
+                        doc["id"], partition_key=doc["session_id"]
+                    )
                 except Exception as e:
-                    self.logger.warning("Failed deleting current team doc %s: %s", doc.get("id"), e)
+                    self.logger.warning(
+                        "Failed deleting current team doc %s: %s", doc.get("id"), e
+                    )
 
         return True
 
@@ -457,9 +446,13 @@ async def delete_plan_by_plan_id(self, plan_id: str) -> bool:
         if items:
             async for doc in items:
                 try:
-                    await self.container.delete_item(doc["id"], partition_key=doc["session_id"])
+                    await self.container.delete_item(
+                        doc["id"], partition_key=doc["session_id"]
+                    )
                 except Exception as e:
-                    self.logger.warning("Failed deleting current team doc %s: %s", doc.get("id"), e)
+                    self.logger.warning(
+                        "Failed deleting current team doc %s: %s", doc.get("id"), e
+                    )
 
         return True
 
@@ -471,7 +464,6 @@ async def update_mplan(self, mplan: messages.MPlan) -> None:
         """Update a team configuration in the database."""
         await self.update_item(mplan)
 
-
     async def get_mplan(self, plan_id: str) -> Optional[messages.MPlan]:
         """Retrieve a mplan configuration by mplan_id."""
         query = "SELECT * FROM c WHERE c.plan_id=@plan_id AND c.data_type=@data_type"
@@ -481,7 +473,6 @@ async def get_mplan(self, plan_id: str) -> Optional[messages.MPlan]:
         ]
         results = await self.query_items(query, parameters, messages.MPlan)
         return results[0] if results else None
-    
 
     async def add_agent_message(self, message: AgentMessageData) -> None:
         """Add an agent message to the database."""
@@ -499,4 +490,4 @@ async def get_agent_messages(self, plan_id: str) -> List[AgentMessageData]:
             {"name": "@data_type", "value": DataType.m_plan_message},
         ]
 
-        return await self.query_items(query, parameters, AgentMessageData)
+        return await self.query_items(query, parameters, AgentMessageData)