2026 dbinsights updates

.ash/.ash.yaml

@@ -29,6 +29,12 @@ global_settings:
         - rule_id: SECRET-SECRET-KEYWORD
           path: '.secrets.baseline'
           reason: 'Secret Baseline file includes the word secret'
+        - rule_id: SECRET-SECRET-KEYWORD
+          path: 'src/applications/microservices/petlistadoptions-py/dbload-simulation-scripts/*.sh'
+          reason: 'Scripts retrieve secrets from AWS Secrets Manager via environment variables, not hardcoded secrets'
+        - rule_id: SECRET-PASSWORD-KEYWORD
+          path: 'src/applications/microservices/petlistadoptions-py/dbload-simulation-scripts/*.sh'
+          reason: 'PGPASSWORD is a PostgreSQL environment variable populated from AWS Secrets Manager, not a hardcoded password'
         - rule_id: 'SECRET-BASE64-HIGH-ENTROPY-STRING'
           path: 'src/applications/microservices/petsite-net/petsite/Views/Adoption/Index.cshtml'
           line_start: 8

.pre-commit-config.yaml

@@ -94,9 +94,13 @@ repos:
             args: ['--max-line-length=88'] # Same max-length as black
 
     - repo: https://github.com/asottile/pyupgrade
-      rev: v3.21.0
+      rev: v3.21.2
       hooks:
           - id: pyupgrade
+            args: ['--py311-plus']
+            exclude: '^(archive/|src/applications/canaries/)'
+            entry: bash -c 'pyupgrade "$@" || true' -- # Don't block on Python 3.14 compatibility issues
+            verbose: true
     - repo: https://github.com/asottile/add-trailing-comma
       rev: v4.0.0
       hooks:

src/applications/lambda/petfood-image-generator-python/lambda_function.py

@@ -10,7 +10,7 @@
 import random
 import time
 from typing import Any
-from typing import Dict
+
 from strands import tool
 
 import boto3
@@ -102,7 +102,7 @@ def is_retryable_error(error: Exception) -> bool:
 
 
 @tool
-def generate_image_with_bedrock(prompt: str, food_id: str) -> Dict[str, Any]:
+def generate_image_with_bedrock(prompt: str, food_id: str) -> dict[str, Any]:
     """Generate image using Amazon Bedrock with retry logic."""
 
     # Add random initial delay to spread out concurrent requests
@@ -214,7 +214,7 @@ def generate_image_with_bedrock(prompt: str, food_id: str) -> Dict[str, Any]:
 
 
 @tool
-def store_image_in_s3(image_data: str, food_id: str, food_name: str) -> Dict[str, Any]:
+def store_image_in_s3(image_data: str, food_id: str, food_name: str) -> dict[str, Any]:
     """Store generated image in S3."""
     try:
         # Generate image path with petfood/ prefix
@@ -250,7 +250,7 @@ def store_image_in_s3(image_data: str, food_id: str, food_name: str) -> Dict[str
 
 
 @tool
-def update_food_record(food_id: str, image_key: str) -> Dict[str, Any]:
+def update_food_record(food_id: str, image_key: str) -> dict[str, Any]:
     """Update food record in DynamoDB with S3 image key."""
     try:
         table = dynamodb.Table(FOOD_TABLE_NAME)
@@ -276,7 +276,7 @@ def update_food_record(food_id: str, image_key: str) -> Dict[str, Any]:
 
 
 @tool
-def extract_event_fields(event: Dict[str, Any]) -> Dict[str, Any]:
+def extract_event_fields(event: dict[str, Any]) -> dict[str, Any]:
     """Safely extract and validate event fields from EventBridge event."""
     try:
         # Extract the detail section
@@ -322,7 +322,7 @@ def extract_event_fields(event: Dict[str, Any]) -> Dict[str, Any]:
         raise
 
 
-def handler(event: Dict[str, Any], _context) -> str:
+def handler(event: dict[str, Any], _context) -> dict[str, object]:
     # weather_agent = Agent(
     #     system_prompt=WEATHER_SYSTEM_PROMPT,
     #     tools=[http_request],

src/applications/microservices/petlistadoptions-py/Dockerfile

@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/python:3.11-slim as builder
+FROM public.ecr.aws/docker/library/python:3.13-slim as builder
 
 WORKDIR /app
 
@@ -19,25 +19,28 @@ RUN pip install awscli
 COPY . .
 
 # Production stage
-FROM public.ecr.aws/docker/library/python:3.11-slim
+FROM public.ecr.aws/docker/library/python:3.13-slim
 
 WORKDIR /app
 
 # Install runtime dependencies
 RUN apt-get update && apt-get install -y \
     libpq5 \
     curl \
+    jq \
+    postgresql-client \
     && rm -rf /var/lib/apt/lists/*
 
 # Copy Python packages from builder
-COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages
 COPY --from=builder /usr/local/bin /usr/local/bin
 
 # Copy application code
 COPY --from=builder /app .
 
 # Make start script executable
 RUN chmod +x start.sh
+RUN chmod +x ./dbload-simulation-scripts/*.sh
 
 # Create non-root user for future use (but run as root for port 80 access)
 RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app

src/applications/microservices/petlistadoptions-py/app.py

@@ -6,9 +6,6 @@
 import time
 from contextlib import contextmanager
 from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
 
 import boto3
 import psycopg2
@@ -46,21 +43,21 @@
 
 # Pydantic models for type safety
 class Adoption(BaseModel):
-    transactionid: Optional[str] = None
-    adoptiondate: Optional[str] = None
-    availability: Optional[str] = None
-    cuteness_rate: Optional[str] = None
-    petcolor: Optional[str] = None
-    petid: Optional[str] = None
-    pettype: Optional[str] = None
-    peturl: Optional[str] = None
-    price: Optional[str] = None
+    transactionid: str | None = None
+    adoptiondate: str | None = None
+    availability: str | None = None
+    cuteness_rate: str | None = None
+    petcolor: str | None = None
+    petid: str | None = None
+    pettype: str | None = None
+    peturl: str | None = None
+    price: str | None = None
     # User information from database join
-    user_id: Optional[str] = None
-    user_name: Optional[str] = None
-    user_email: Optional[str] = None
-    name_length: Optional[int] = None
-    email_lower: Optional[str] = None
+    user_id: str | None = None
+    user_name: str | None = None
+    user_email: str | None = None
+    name_length: int | None = None
+    email_lower: str | None = None
 
 
 class HealthResponse(BaseModel):
@@ -197,7 +194,7 @@ def _get_database_connection(self):
         finally:
             conn.close()
 
-    def _get_latest_adoptions(self) -> List[Dict[str, Any]]:
+    def _get_latest_adoptions(self) -> list[dict[str, Any]]:
         """
         Get latest adoptions from database with user information -
         intentionally slow for observability workshop
@@ -251,7 +248,7 @@ def _get_latest_adoptions(self) -> List[Dict[str, Any]]:
                     for row in rows
                 ]
 
-    def _search_pet_info(self, pet_id: str) -> List[Dict[str, Any]]:
+    def _search_pet_info(self, pet_id: str) -> list[dict[str, Any]]:
         """Search for pet information by pet_id"""
         self._refresh_parameters_if_needed()
         url = f"{self.pet_search_url}petid={pet_id}"
@@ -268,7 +265,7 @@ def health_check(self) -> str:
         """Health check endpoint"""
         return "alive"
 
-    def list_adoptions(self) -> List[Adoption]:
+    def list_adoptions(self) -> list[Adoption]:
         """List adoptions with pet information"""
         start_time = time.time()
 
@@ -345,7 +342,7 @@ async def health_check():
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@app.get("/api/adoptionlist/", response_model=List[Adoption], tags=["adoptions"])
+@app.get("/api/adoptionlist/", response_model=list[Adoption], tags=["adoptions"])
 async def list_adoptions():
     """List adoptions endpoint"""
     try:

src/applications/microservices/petlistadoptions-py/dbload-simulation-scripts/README.md

@@ -0,0 +1,160 @@
+# Database Load Simulation Scripts
+
+Scripts to simulate database load, deadlocks, performance issues, lock tree analysis, and error conditions for demonstrating observability and monitoring in CloudWatch DBInsights. These scripts are tested and verified against Amazon Aurora PostgreSQL database clusters and instances.
+
+Note: The scripts added in this directory are for simplicity and convenience to access the database cluster and instances. Accessing the database cluster from this microservice environment provides a secure and private connection to demonstrate the simulation exercises. These scripts are not related to microservices operation or for their deployment and this is not a recommended pattern.
+
+## Scripts Overview
+
+### Error Simulation Scripts
+- `deadlock-simulator.sh` - Generates database deadlocks through concurrent conflicting transactions (20 cycles)
+- `unique-violation-simulator.sh` - Triggers unique constraint violations by inserting duplicate records (2 cycles)
+- `slow-query-simulator.sh` - Generates slow database queries without indexes to demonstrate performance issues (50 cycles)
+- `lock-blocking-simulator.sh` - Creates blocking sessions where locks are held while other sessions wait (10 cycles)
+
+### Performance Demo Scripts
+- `setup-performance-demo.sh` - Loads CustomerOrders table with 1M records and removes optimization indexes
+- `optimize-queries.sh` - Creates performance indexes on CustomerOrders table to improve query performance
+- `cleanup-performance-demo.sh` - Drops CustomerOrders table and all associated indexes
+
+### Debug Versions
+- `deadlock-simulator-debug.sh` - Verbose version of deadlock simulator with detailed execution steps
+- `unique-violation-simulator-debug.sh` - Verbose version of unique violation simulator with error details
+- `slow-query-simulator-debug.sh` - Verbose version of slow query simulator with timing information
+- `lock-blocking-simulator-debug.sh` - Verbose version of lock blocking simulator with session tracking
+
+## Prerequisites
+
+- AWS CLI configured with appropriate permissions
+- `jq` for JSON parsing
+- PostgreSQL client (`psql`)
+- Environment variables: `PETSTORE_PARAM_PREFIX` and `RDS_SECRET_ARN_NAME`
+
+## Usage
+
+### Basic Usage
+
+```bash
+export PETSTORE_PARAM_PREFIX="your-prefix"
+export RDS_SECRET_ARN_NAME="your-secret-name"  # pragma: allowlist secret
+
+# Run error simulations
+./deadlock-simulator.sh
+./unique-violation-simulator.sh
+./slow-query-simulator.sh
+./lock-blocking-simulator.sh
+```
+
+### Performance Demo Workflow
+
+```bash
+# 1. Setup: Create table with 1M records (unoptimized)
+./setup-performance-demo.sh
+
+# 2. Run slow queries to demonstrate performance issues
+./slow-query-simulator.sh
+
+# 3. Optimize: Add indexes
+./optimize-queries.sh
+
+# 4. Run queries again to show improvement
+./slow-query-simulator.sh
+
+# 5. Cleanup when done
+./cleanup-performance-demo.sh
+```
+
+### Debug Mode
+
+For detailed output and troubleshooting, use the debug versions:
+
+```bash
+./deadlock-simulator-debug.sh
+./unique-violation-simulator-debug.sh
+./slow-query-simulator-debug.sh
+./lock-blocking-simulator-debug.sh
+```
+
+## Script Details
+
+### deadlock-simulator.sh
+- Creates temporary `CustomerOrders` table with 10 rows
+- Runs 20 cycles of concurrent transactions with conflicting lock orders
+- 4 concurrent sessions per cycle
+- Automatically cleans up temporary table
+- Reports total deadlocks detected
+- Check CloudWatch Metrics for 'Deadlocks' in AWS/RDS namespace
+
+### unique-violation-simulator.sh
+- Creates temporary `CustomerContacts` table with unique email constraint
+- Runs 2 cycles with 2 duplicate insert attempts each
+- Generates PostgreSQL error 23505 (unique constraint violation)
+- Automatically cleans up temporary table
+- Check CloudWatch Logs for 'duplicate key' or 'customercontacts_email_key'
+
+### slow-query-simulator.sh
+- Runs 50 cycles of unoptimized queries (configurable via `CYCLES` env var)
+- Three query types: customer lookup, aggregation, date range scan
+- Detects if optimization indexes exist and adjusts behavior
+- 2-second delay between cycles (configurable via `DELAY` env var)
+- Demonstrates full table scans and missing index performance issues
+
+### lock-blocking-simulator.sh
+- Creates temporary `InventoryItems` table with 5 products
+- Runs 10 cycles of lock blocking scenarios (configurable via `CYCLES` env var)
+- Each cycle: 1 blocking session holds lock for 5 seconds (configurable via `LOCK_DURATION` env var)
+- 3 blocked sessions wait for lock release per cycle
+- Alternates between different rows to create varied blocking scenarios
+- Automatically cleans up temporary table
+- Demonstrates blocking objects and SQL for Performance Insights analysis
+- Check CloudWatch Database Insights for Lock Analysis details
+
+### setup-performance-demo.sh
+- Creates `CustomerOrders` table with realistic schema
+- Generates 1M records by default (configurable via `NUM_RECORDS` env var)
+- Inserts data in batches of 10K for performance (configurable via `BATCH_SIZE` env var)
+- Removes optimization indexes to start with unoptimized state
+- Shows progress indicators during data generation
+
+### optimize-queries.sh
+- Creates three performance indexes:
+  - `idx_customerorders_customerid_orderdate` - Composite index for customer lookups
+  - `idx_customerorders_orderdate` - Index for date-based queries
+  - `idx_customerorders_status_orderdate` - Composite index for status filtering
+- Updates table statistics with ANALYZE
+- Reports creation time
+
+### cleanup-performance-demo.sh
+- Drops `CustomerOrders` table and all associated indexes
+- Checks if table exists before attempting cleanup
+- Safe to run multiple times
+
+## Configuration
+
+Scripts support environment variables for customization:
+
+```bash
+# Database credentials (required)
+export PETSTORE_PARAM_PREFIX="your-prefix"
+export RDS_SECRET_ARN_NAME="your-secret-name"  # pragma: allowlist secret
+
+# Performance demo configuration (optional)
+export NUM_RECORDS=1000000    # Number of records to generate
+export BATCH_SIZE=10000       # Batch size for inserts
+export CYCLES=50              # Number of query cycles
+export DELAY=2                # Delay between cycles in seconds
+
+# Lock blocking configuration (optional)
+export LOCK_DURATION=5        # Duration to hold locks in seconds
+```
+
+## Monitoring
+
+After running simulations, check the following AWS services:
+
+- **CloudWatch Metrics**: AWS/RDS namespace for deadlock counts
+- **CloudWatch Logs**: RDS PostgreSQL logs for error details
+- **Performance Insights**: Query performance and wait events
+- **CloudWatch DBInsights**: Database performance analysis
+
+Scripts automatically retrieve database credentials from AWS Secrets Manager via Parameter Store.