Skip to content

Commit bd598ad

Browse files
adamdougaladamdougal
authored
Allow the setting of log level by env vars (#642)
* Allow the setting of log level by env vars Required by #637 * Allow the setting of log level by env vars Required by #637 * Add loglevel to env.sample * Allow log level to be set via bicep * Set log level in admin app * Correctly configure logs in web app * Ensure log level is set for functions * Remove duplicated tests * Fix main.json after merge * Reduce verbose azure logs
1 parent 2b63ecf commit bd598ad

17 files changed

+89
-26
lines changed

‎.env.sample

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,4 @@ AZURE_SPEECH_SERVICE_REGION=
5555
AZURE_AUTH_TYPE=keys
5656
USE_KEY_VAULT=true
5757
AZURE_KEY_VAULT_ENDPOINT=
58+
LOGLEVEL=INFO

‎code/app.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
import os
2+
import logging
23
from azure.monitor.opentelemetry import configure_azure_monitor
34
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
45

6+
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO").upper())
7+
# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
8+
logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
59
# We cannot use EnvHelper here as Application Insights should be configured first
610
# for instrumentation to work correctly
711
if os.getenv("APPINSIGHTS_ENABLED", "false").lower() == "true":

‎code/backend/Admin.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,16 @@
1010

1111
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
1212

13+
logging.basicConfig(level=os.getenv("LOGLEVEL", "INFO").upper())
14+
# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
15+
logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
1316
# We cannot use EnvHelper here as Application Insights needs to be configured first
1417
# for instrumentation to work correctly
1518
if os.getenv("APPINSIGHTS_ENABLED", "false").lower() == "true":
1619
configure_azure_monitor()
1720

18-
logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
19-
logging.WARNING
20-
)
21+
logger = logging.getLogger(__name__)
22+
logger.debug("Starting admin app")
2123

2224

2325
st.set_page_config(

‎code/backend/batch/AddURLEmbeddings.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@
33
import azure.functions as func
44
import sys
55

6+
from utilities.helpers.EnvHelper import EnvHelper
67
from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
78
from utilities.helpers.ConfigHelper import ConfigHelper
89

910
sys.path.append("..")
1011

1112
bp_add_url_embeddings = func.Blueprint()
13+
env_helper: EnvHelper = EnvHelper()
1214

1315
logger = logging.getLogger(__name__)
16+
logger.setLevel(env_helper.LOGLEVEL)
1417

1518

1619
@bp_add_url_embeddings.route(route="AddURLEmbeddings")

‎code/backend/batch/BatchPushResults.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,18 @@
44
from urllib.parse import urlparse
55
import sys
66

7+
from utilities.helpers.EnvHelper import EnvHelper
78
from utilities.helpers.AzureBlobStorageHelper import AzureBlobStorageClient
89
from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
910
from utilities.helpers.ConfigHelper import ConfigHelper
1011

1112
sys.path.append("..")
1213

1314
bp_batch_push_results = func.Blueprint()
15+
env_helper: EnvHelper = EnvHelper()
1416

1517
logger = logging.getLogger(__name__)
18+
logger.setLevel(env_helper.LOGLEVEL)
1619

1720

1821
def _get_file_name_from_message(msg: func.QueueMessage) -> str:

‎code/backend/batch/BatchStartProcessing.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
env_helper: EnvHelper = EnvHelper()
1414

1515
logger = logging.getLogger(__name__)
16+
logger.setLevel(env_helper.LOGLEVEL)
1617

1718

1819
@bp_batch_start_processing.route(route="BatchStartProcessing")

‎code/backend/batch/GetConversationResponse.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
env_helper: EnvHelper = EnvHelper()
1212

1313
logger = logging.getLogger(__name__)
14+
logger.setLevel(env_helper.LOGLEVEL)
1415

1516

1617
@bp_get_conversation_response.route(route="GetConversationResponse")

‎code/backend/batch/function_app.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
import logging
2+
import os
13
import azure.functions as func
24
from AddURLEmbeddings import bp_add_url_embeddings
35
from BatchPushResults import bp_batch_push_results
46
from BatchStartProcessing import bp_batch_start_processing
57
from GetConversationResponse import bp_get_conversation_response
68
from azure.monitor.opentelemetry import configure_azure_monitor
79

10+
# Raising the azure log level to WARN as it is too verbose - https://github.com/Azure/azure-sdk-for-python/issues/9422
11+
logging.getLogger("azure").setLevel(os.environ.get("LOGLEVEL_AZURE", "WARN").upper())
812
configure_azure_monitor()
913

1014
app = func.FunctionApp(

‎code/backend/batch/utilities/helpers/EnvHelper.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ def __init__(self, **kwargs) -> None:
1414
# Wrapper for Azure Key Vault
1515
self.secretHelper = SecretHelper()
1616

17+
self.LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
18+
1719
# Azure Search
1820
self.AZURE_SEARCH_SERVICE = os.getenv("AZURE_SEARCH_SERVICE", "")
1921
self.AZURE_SEARCH_INDEX = os.getenv("AZURE_SEARCH_INDEX", "")

‎code/backend/pages/01_Ingest_Data.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import traceback
66
import chardet
77
from datetime import datetime, timedelta
8-
import logging
98
import requests
109
from azure.identity import DefaultAzureCredential
1110
from azure.storage.blob import (
@@ -22,9 +21,6 @@
2221
sys.path.append(path.join(path.dirname(__file__), ".."))
2322
env_helper: EnvHelper = EnvHelper()
2423

25-
logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
26-
logging.WARNING
27-
)
2824
st.set_page_config(
2925
page_title="Ingest Data",
3026
page_icon=path.join("images", "favicon.ico"),

‎code/backend/pages/02_Explore_Data.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
import os
33
import json
44
import traceback
5-
import logging
65
import pandas as pd
76
import sys
87
from batch.utilities.helpers.AzureSearchHelper import AzureSearchHelper
@@ -12,9 +11,6 @@
1211

1312
load_dotenv()
1413

15-
logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
16-
logging.WARNING
17-
)
1814
st.set_page_config(
1915
page_title="Explore Data",
2016
page_icon=os.path.join("images", "favicon.ico"),

‎code/backend/pages/03_Delete_Data.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import streamlit as st
22
import os
33
import traceback
4-
import logging
54
import sys
65
from batch.utilities.helpers.AzureSearchHelper import AzureSearchHelper
76
from dotenv import load_dotenv
@@ -10,9 +9,6 @@
109

1110
load_dotenv()
1211

13-
logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
14-
logging.WARNING
15-
)
1612
st.set_page_config(
1713
page_title="Delete Data",
1814
page_icon=os.path.join("images", "favicon.ico"),

‎code/backend/pages/04_Configuration.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import streamlit as st
22
import os
33
import traceback
4-
import logging
54
from dotenv import load_dotenv
65
import sys
76
from batch.utilities.helpers.ConfigHelper import ConfigHelper
@@ -10,9 +9,6 @@
109

1110
load_dotenv()
1211

13-
logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
14-
logging.WARNING
15-
)
1612
st.set_page_config(
1713
page_title="Configure Prompts",
1814
page_icon=os.path.join("images", "favicon.ico"),

‎code/create_app.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,8 @@ def create_app():
298298
app = Flask(__name__)
299299
env_helper: EnvHelper = EnvHelper()
300300

301+
logger.debug("Starting web app")
302+
301303
@app.route("/", defaults={"path": "index.html"})
302304
@app.route("/<path:path>")
303305
def static_file(path):

‎code/tests/utilities/test_EnvHelper.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,22 @@ def test_keys_are_unset_when_auth_type_rbac(monkeypatch: MonkeyPatch):
7272
assert env_helper.AZURE_SEARCH_KEY is None
7373
assert env_helper.AZURE_OPENAI_API_KEY == ""
7474
assert env_helper.AZURE_SPEECH_KEY is None
75+
76+
77+
def test_sets_default_log_level_when_unset():
78+
# when
79+
env_helper = EnvHelper()
80+
81+
# then
82+
assert env_helper.LOGLEVEL == "INFO"
83+
84+
85+
def test_uses_and_uppercases_log_level_when_set(monkeypatch: MonkeyPatch):
86+
# given
87+
monkeypatch.setenv("LOGLEVEL", "deBug")
88+
89+
# when
90+
env_helper = EnvHelper()
91+
92+
# then
93+
assert env_helper.LOGLEVEL == "DEBUG"

‎infra/main.bicep

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,15 @@ param authType string = 'keys'
177177
])
178178
param hostingModel string = 'container'
179179

180+
@allowed([
181+
'CRITICAL'
182+
'ERROR'
183+
'WARN'
184+
'INFO'
185+
'DEBUG'
186+
])
187+
param logLevel string = 'INFO'
188+
180189
var blobContainerName = 'documents'
181190
var queueName = 'doc-processing'
182191
var clientKey = '${uniqueString(guid(subscription().id, deployment().name))}${newGuidString}'
@@ -382,6 +391,7 @@ module web './app/web.bicep' = if (hostingModel == 'code') {
382391
AZURE_SPEECH_SERVICE_NAME: speechServiceName
383392
AZURE_SPEECH_SERVICE_REGION: location
384393
ORCHESTRATION_STRATEGY: orchestrationStrategy
394+
LOGLEVEL: logLevel
385395
}
386396
}
387397
}
@@ -442,6 +452,7 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') {
442452
AZURE_SPEECH_SERVICE_NAME: speechServiceName
443453
AZURE_SPEECH_SERVICE_REGION: location
444454
ORCHESTRATION_STRATEGY: orchestrationStrategy
455+
LOGLEVEL: logLevel
445456
}
446457
}
447458
}
@@ -503,6 +514,7 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') {
503514
DOCUMENT_PROCESSING_QUEUE_NAME: queueName
504515
FUNCTION_KEY: clientKey
505516
ORCHESTRATION_STRATEGY: orchestrationStrategy
517+
LOGLEVEL: logLevel
506518
}
507519
}
508520
}
@@ -563,6 +575,7 @@ module adminweb_docker './app/adminweb.bicep' = if (hostingModel == 'container')
563575
DOCUMENT_PROCESSING_QUEUE_NAME: queueName
564576
FUNCTION_KEY: clientKey
565577
ORCHESTRATION_STRATEGY: orchestrationStrategy
578+
LOGLEVEL: logLevel
566579
}
567580
}
568581
}
@@ -639,6 +652,7 @@ module function './app/function.bicep' = if (hostingModel == 'code') {
639652
AZURE_SEARCH_SERVICE: 'https://${azureAISearchName}.search.windows.net'
640653
DOCUMENT_PROCESSING_QUEUE_NAME: queueName
641654
ORCHESTRATION_STRATEGY: orchestrationStrategy
655+
LOGLEVEL: logLevel
642656
}
643657
}
644658
}
@@ -682,6 +696,7 @@ module function_docker './app/function.bicep' = if (hostingModel == 'container')
682696
AZURE_SEARCH_SERVICE: 'https://${azureAISearchName}.search.windows.net'
683697
DOCUMENT_PROCESSING_QUEUE_NAME: queueName
684698
ORCHESTRATION_STRATEGY: orchestrationStrategy
699+
LOGLEVEL: logLevel
685700
}
686701
}
687702
}
@@ -843,3 +858,4 @@ output USE_KEY_VAULT bool = useKeyVault
843858
output AZURE_APP_SERVICE_HOSTING_MODEL string = hostingModel
844859
output FRONTEND_WEBSITE_NAME string = hostingModel == 'code' ? web.outputs.FRONTEND_API_URI : web_docker.outputs.FRONTEND_API_URI
845860
output ADMIN_WEBSITE_NAME string = hostingModel == 'code' ? adminweb.outputs.WEBSITE_ADMIN_URI : adminweb_docker.outputs.WEBSITE_ADMIN_URI
861+
output LOGLEVEL string = logLevel

‎infra/main.json

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"_generator": {
66
"name": "bicep",
77
"version": "0.26.54.24096",
8-
"templateHash": "7715045314239280418"
8+
"templateHash": "15295378206128435887"
99
}
1010
},
1111
"parameters": {
@@ -372,6 +372,17 @@
372372
"metadata": {
373373
"description": "Hosting model for the web apps. Containers are prebuilt and can be deployed faster, but code allows for more customization."
374374
}
375+
},
376+
"logLevel": {
377+
"type": "string",
378+
"defaultValue": "INFO",
379+
"allowedValues": [
380+
"CRITICAL",
381+
"ERROR",
382+
"WARN",
383+
"INFO",
384+
"DEBUG"
385+
]
375386
}
376387
},
377388
"variables": {
@@ -1506,7 +1517,8 @@
15061517
"AZURE_SEARCH_URL_COLUMN": "[parameters('azureSearchUrlColumn')]",
15071518
"AZURE_SPEECH_SERVICE_NAME": "[parameters('speechServiceName')]",
15081519
"AZURE_SPEECH_SERVICE_REGION": "[parameters('location')]",
1509-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
1520+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
1521+
"LOGLEVEL": "[parameters('logLevel')]"
15101522
}
15111523
}
15121524
},
@@ -2359,7 +2371,8 @@
23592371
"AZURE_SEARCH_URL_COLUMN": "[parameters('azureSearchUrlColumn')]",
23602372
"AZURE_SPEECH_SERVICE_NAME": "[parameters('speechServiceName')]",
23612373
"AZURE_SPEECH_SERVICE_REGION": "[parameters('location')]",
2362-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
2374+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
2375+
"LOGLEVEL": "[parameters('logLevel')]"
23632376
}
23642377
}
23652378
},
@@ -3215,7 +3228,8 @@
32153228
"BACKEND_URL": "[format('https://{0}.azurewebsites.net', parameters('functionName'))]",
32163229
"DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]",
32173230
"FUNCTION_KEY": "[variables('clientKey')]",
3218-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
3231+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
3232+
"LOGLEVEL": "[parameters('logLevel')]"
32193233
}
32203234
}
32213235
},
@@ -4137,7 +4151,8 @@
41374151
"BACKEND_URL": "[format('https://{0}-docker.azurewebsites.net', parameters('functionName'))]",
41384152
"DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]",
41394153
"FUNCTION_KEY": "[variables('clientKey')]",
4140-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
4154+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
4155+
"LOGLEVEL": "[parameters('logLevel')]"
41414156
}
41424157
}
41434158
},
@@ -6780,7 +6795,8 @@
67806795
"AZURE_SEARCH_INDEX": "[parameters('azureSearchIndex')]",
67816796
"AZURE_SEARCH_SERVICE": "[format('https://{0}.search.windows.net', parameters('azureAISearchName'))]",
67826797
"DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]",
6783-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
6798+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
6799+
"LOGLEVEL": "[parameters('logLevel')]"
67846800
}
67856801
}
67866802
},
@@ -7996,7 +8012,8 @@
79968012
"AZURE_SEARCH_INDEX": "[parameters('azureSearchIndex')]",
79978013
"AZURE_SEARCH_SERVICE": "[format('https://{0}.search.windows.net', parameters('azureAISearchName'))]",
79988014
"DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]",
7999-
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]"
8015+
"ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]",
8016+
"LOGLEVEL": "[parameters('logLevel')]"
80008017
}
80018018
}
80028019
},
@@ -10261,6 +10278,10 @@
1026110278
"ADMIN_WEBSITE_NAME": {
1026210279
"type": "string",
1026310280
"value": "[if(equals(parameters('hostingModel'), 'code'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('adminWebsiteName')), '2022-09-01').outputs.WEBSITE_ADMIN_URI.value, reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', format('{0}-docker', parameters('adminWebsiteName'))), '2022-09-01').outputs.WEBSITE_ADMIN_URI.value)]"
10281+
},
10282+
"LOGLEVEL": {
10283+
"type": "string",
10284+
"value": "[parameters('logLevel')]"
1026410285
}
1026510286
}
1026610287
}

0 commit comments

Comments
 (0)