Skip to content

Commit cf4beff

Browse files
authored
feat(snowflake-driver): support DefaultAzureCredential and Managed Indentity auth for export bucket (#8792)
1 parent d77a512 commit cf4beff

File tree

5 files changed

+202
-21
lines changed

5 files changed

+202
-21
lines changed

packages/cubejs-backend-shared/src/env.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -782,6 +782,45 @@ const variables: Record<string, (...args: any) => any> = {
782782
]
783783
),
784784

785+
/**
786+
* Azure Client ID for the Azure based export bucket storage.
787+
*/
788+
dbExportBucketAzureClientId: ({
789+
dataSource,
790+
}: {
791+
dataSource: string,
792+
}) => (
793+
process.env[
794+
keyByDataSource('CUBEJS_DB_EXPORT_BUCKET_AZURE_CLIENT_ID', dataSource)
795+
]
796+
),
797+
798+
/**
799+
* Azure Federated Token File Path for the Azure based export bucket storage.
800+
*/
801+
dbExportBucketAzureTokenFilePAth: ({
802+
dataSource,
803+
}: {
804+
dataSource: string,
805+
}) => (
806+
process.env[
807+
keyByDataSource('CUBEJS_DB_EXPORT_BUCKET_AZURE_FEDERATED_TOKEN_FILE', dataSource)
808+
]
809+
),
810+
811+
/**
812+
* Azure Tenant ID for the Azure based export bucket storage.
813+
*/
814+
dbExportBucketAzureTenantId: ({
815+
dataSource,
816+
}: {
817+
dataSource: string,
818+
}) => (
819+
process.env[
820+
keyByDataSource('CUBEJS_DB_EXPORT_BUCKET_AZURE_TENANT_ID', dataSource)
821+
]
822+
),
823+
785824
/**
786825
* Export bucket options for Integration based.
787826
*/

packages/cubejs-base-driver/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
"dependencies": {
3232
"@aws-sdk/client-s3": "^3.49.0",
3333
"@aws-sdk/s3-request-presigner": "^3.49.0",
34+
"@azure/identity": "^4.4.1",
3435
"@azure/storage-blob": "^12.9.0",
3536
"@cubejs-backend/shared": "^0.36.5",
3637
"@google-cloud/storage": "^7.13.0",

packages/cubejs-base-driver/src/BaseDriver.ts

Lines changed: 73 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ import {
2626
SASProtocol,
2727
generateBlobSASQueryParameters,
2828
} from '@azure/storage-blob';
29+
import {
30+
DefaultAzureCredential,
31+
} from '@azure/identity';
2932

3033
import { cancelCombinator } from './utils';
3134
import {
@@ -52,9 +55,30 @@ import {
5255
ForeignKeysQueryResult,
5356
} from './driver.interface';
5457

58+
/**
59+
* @see {@link DefaultAzureCredential} constructor options
60+
*/
5561
export type AzureStorageClientConfig = {
56-
azureKey: string,
62+
azureKey?: string,
5763
sasToken?: string,
64+
/**
65+
* The client ID of a Microsoft Entra app registration.
66+
* In case of DefaultAzureCredential flow if it is omitted
67+
* the Azure library will try to use the AZURE_CLIENT_ID env
68+
*/
69+
clientId?: string,
70+
/**
71+
* ID of the application's Microsoft Entra tenant. Also called its directory ID.
72+
* In case of DefaultAzureCredential flow if it is omitted
73+
* the Azure library will try to use the AZURE_TENANT_ID env
74+
*/
75+
tenantId?: string,
76+
/**
77+
* The path to a file containing a Kubernetes service account token that authenticates the identity.
78+
* In case of DefaultAzureCredential flow if it is omitted
79+
* the Azure library will try to use the AZURE_FEDERATED_TOKEN_FILE env
80+
*/
81+
tokenFilePath?: string,
5882
};
5983

6084
export type GoogleStorageClientConfig = {
@@ -730,9 +754,52 @@ export abstract class BaseDriver implements DriverInterface {
730754
const parts = bucketName.split('.blob.core.windows.net/');
731755
const account = parts[0];
732756
const container = parts[1].split('/')[0];
733-
const credential = new StorageSharedKeyCredential(account, azureConfig.azureKey);
757+
let credential: StorageSharedKeyCredential | DefaultAzureCredential;
758+
let blobServiceClient: BlobServiceClient;
759+
let getSas;
760+
761+
if (azureConfig.azureKey) {
762+
credential = new StorageSharedKeyCredential(account, azureConfig.azureKey);
763+
getSas = async (name: string, startsOn: Date, expiresOn: Date) => generateBlobSASQueryParameters(
764+
{
765+
containerName: container,
766+
blobName: name,
767+
permissions: ContainerSASPermissions.parse('r'),
768+
startsOn,
769+
expiresOn,
770+
protocol: SASProtocol.Https,
771+
version: '2020-08-04',
772+
},
773+
credential as StorageSharedKeyCredential
774+
).toString();
775+
} else {
776+
const opts = {
777+
tenantId: azureConfig.tenantId,
778+
clientId: azureConfig.clientId,
779+
tokenFilePath: azureConfig.tokenFilePath,
780+
};
781+
credential = new DefaultAzureCredential(opts);
782+
getSas = async (name: string, startsOn: Date, expiresOn: Date) => {
783+
// getUserDelegationKey works only for authorization with Microsoft Entra ID
784+
const userDelegationKey = await blobServiceClient.getUserDelegationKey(startsOn, expiresOn);
785+
return generateBlobSASQueryParameters(
786+
{
787+
containerName: container,
788+
blobName: name,
789+
permissions: ContainerSASPermissions.parse('r'),
790+
startsOn,
791+
expiresOn,
792+
protocol: SASProtocol.Https,
793+
version: '2020-08-04',
794+
},
795+
userDelegationKey,
796+
account,
797+
).toString();
798+
};
799+
}
800+
734801
const url = `https://${account}.blob.core.windows.net`;
735-
const blobServiceClient = azureConfig.sasToken ?
802+
blobServiceClient = azureConfig.sasToken ?
736803
new BlobServiceClient(`${url}?${azureConfig.sasToken}`) :
737804
new BlobServiceClient(url, credential);
738805

@@ -741,19 +808,9 @@ export abstract class BaseDriver implements DriverInterface {
741808
const blobsList = containerClient.listBlobsFlat({ prefix: `${tableName}/` });
742809
for await (const blob of blobsList) {
743810
if (blob.name && (blob.name.endsWith('.csv.gz') || blob.name.endsWith('.csv'))) {
744-
const sas = generateBlobSASQueryParameters(
745-
{
746-
containerName: container,
747-
blobName: blob.name,
748-
permissions: ContainerSASPermissions.parse('r'),
749-
startsOn: new Date(new Date().valueOf()),
750-
expiresOn:
751-
new Date(new Date().valueOf() + 1000 * 60 * 60),
752-
protocol: SASProtocol.Https,
753-
version: '2020-08-04',
754-
},
755-
credential,
756-
).toString();
811+
const starts = new Date();
812+
const expires = new Date(starts.valueOf() + 1000 * 60 * 60);
813+
const sas = await getSas(blob.name, starts, expires);
757814
csvFiles.push(`${url}/${container}/${blob.name}?${sas}`);
758815
}
759816
}

packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,9 +133,21 @@ interface SnowflakeDriverExportGCS {
133133
interface SnowflakeDriverExportAzure {
134134
bucketType: 'azure',
135135
bucketName: string,
136-
azureKey: string,
136+
azureKey?: string,
137137
sasToken?: string,
138138
integrationName?: string,
139+
/**
140+
* The client ID of a Microsoft Entra app registration.
141+
*/
142+
clientId?: string,
143+
/**
144+
* ID of the application's Microsoft Entra tenant. Also called its directory ID.
145+
*/
146+
tenantId?: string,
147+
/**
148+
* The path to a file containing a Kubernetes service account token that authenticates the identity.
149+
*/
150+
tokenFilePath?: string,
139151
}
140152

141153
export type SnowflakeDriverExportBucket = SnowflakeDriverExportAWS | SnowflakeDriverExportGCS
@@ -317,12 +329,30 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
317329
// sasToken is optional for azure if storage integration is used
318330
const sasToken = getEnv('dbExportAzureSasToken', { dataSource });
319331

332+
if (!integrationName && !sasToken) {
333+
throw new Error(
334+
'Unsupported exportBucket configuration, some keys are empty: integrationName|sasToken'
335+
);
336+
}
337+
338+
// azureKey is optional if DefaultAzureCredential() is used
339+
const azureKey = getEnv('dbExportBucketAzureKey', { dataSource });
340+
341+
// These 3 options make sense in case you want to authorize to Azure from
342+
// application running in the k8s environment.
343+
const clientId = getEnv('dbExportBucketAzureClientId', { dataSource });
344+
const tenantId = getEnv('dbExportBucketAzureTenantId', { dataSource });
345+
const tokenFilePath = getEnv('dbExportBucketAzureTokenFilePAth', { dataSource });
346+
320347
return {
321348
bucketType,
322349
bucketName: getEnv('dbExportBucket', { dataSource }),
323-
azureKey: getEnv('dbExportBucketAzureKey', { dataSource }),
324-
...(sasToken !== undefined && { sasToken }),
325350
...(integrationName !== undefined && { integrationName }),
351+
...(sasToken !== undefined && { sasToken }),
352+
...(azureKey !== undefined && { azureKey }),
353+
...(clientId !== undefined && { clientId }),
354+
...(tenantId !== undefined && { tenantId }),
355+
...(tokenFilePath !== undefined && { tokenFilePath }),
326356
};
327357
}
328358

@@ -643,11 +673,11 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface {
643673
);
644674
return this.extractFilesFromGCS({ credentials }, bucketName, tableName);
645675
} else if (bucketType === 'azure') {
646-
const { azureKey, sasToken } = (
676+
const { azureKey, sasToken, clientId, tenantId, tokenFilePath } = (
647677
<SnowflakeDriverExportAzure> this.config.exportBucket
648678
);
649679
return this.extractFilesFromAzure(
650-
{ azureKey, sasToken },
680+
{ azureKey, sasToken, clientId, tenantId, tokenFilePath },
651681
bucketName,
652682
tableName,
653683
);

yarn.lock

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1021,6 +1021,19 @@
10211021
"@azure/logger" "^1.0.0"
10221022
tslib "^2.6.2"
10231023

1024+
"@azure/core-client@^1.9.2":
1025+
version "1.9.2"
1026+
resolved "https://registry.yarnpkg.com/@azure/core-client/-/core-client-1.9.2.tgz#6fc69cee2816883ab6c5cdd653ee4f2ff9774f74"
1027+
integrity sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==
1028+
dependencies:
1029+
"@azure/abort-controller" "^2.0.0"
1030+
"@azure/core-auth" "^1.4.0"
1031+
"@azure/core-rest-pipeline" "^1.9.1"
1032+
"@azure/core-tracing" "^1.0.0"
1033+
"@azure/core-util" "^1.6.1"
1034+
"@azure/logger" "^1.0.0"
1035+
tslib "^2.6.2"
1036+
10241037
"@azure/core-http-compat@^2.0.1":
10251038
version "2.1.0"
10261039
resolved "https://registry.yarnpkg.com/@azure/core-http-compat/-/core-http-compat-2.1.0.tgz#a48451c4e9dae7ad0ca85bbd2b98e0f2ae02836e"
@@ -1124,6 +1137,26 @@
11241137
stoppable "^1.1.0"
11251138
tslib "^2.2.0"
11261139

1140+
"@azure/identity@^4.4.1":
1141+
version "4.4.1"
1142+
resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.4.1.tgz#490fa2ad26786229afa36411892bb53dfa3478d3"
1143+
integrity sha512-DwnG4cKFEM7S3T+9u05NstXU/HN0dk45kPOinUyNKsn5VWwpXd9sbPKEg6kgJzGbm1lMuhx9o31PVbCtM5sfBA==
1144+
dependencies:
1145+
"@azure/abort-controller" "^1.0.0"
1146+
"@azure/core-auth" "^1.5.0"
1147+
"@azure/core-client" "^1.9.2"
1148+
"@azure/core-rest-pipeline" "^1.1.0"
1149+
"@azure/core-tracing" "^1.0.0"
1150+
"@azure/core-util" "^1.3.0"
1151+
"@azure/logger" "^1.0.0"
1152+
"@azure/msal-browser" "^3.14.0"
1153+
"@azure/msal-node" "^2.9.2"
1154+
events "^3.0.0"
1155+
jws "^4.0.0"
1156+
open "^8.0.0"
1157+
stoppable "^1.1.0"
1158+
tslib "^2.2.0"
1159+
11271160
"@azure/keyvault-keys@^4.4.0":
11281161
version "4.8.0"
11291162
resolved "https://registry.yarnpkg.com/@azure/keyvault-keys/-/keyvault-keys-4.8.0.tgz#1513b3a187bb3a9a372b5980c593962fb793b2ad"
@@ -1148,13 +1181,25 @@
11481181
dependencies:
11491182
tslib "^2.6.2"
11501183

1184+
"@azure/msal-browser@^3.14.0":
1185+
version "3.25.0"
1186+
resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.25.0.tgz#7ce0949977bc9e0c58319f7090c44fe5537104d4"
1187+
integrity sha512-a0Y7pmSy8SC1s9bvwr+REvyAA1nQcITlZvkElM2gNUPYFTTNUTEdcpg73TmawNucyMdZ9xb/GFcuhrLOqYAzwg==
1188+
dependencies:
1189+
"@azure/msal-common" "14.15.0"
1190+
11511191
"@azure/msal-browser@^3.5.0":
11521192
version "3.10.0"
11531193
resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.10.0.tgz#8925659e8d1a4bd21e389cca4683eb52658c778e"
11541194
integrity sha512-mnmi8dCXVNZI+AGRq0jKQ3YiodlIC4W9npr6FCB9WN6NQT+6rq+cIlxgUb//BjLyzKsnYo+i4LROGeMyU+6v1A==
11551195
dependencies:
11561196
"@azure/msal-common" "14.7.1"
11571197

1198+
"@azure/msal-common@14.15.0":
1199+
version "14.15.0"
1200+
resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.15.0.tgz#0e27ac0bb88fe100f4f8d1605b64d5c268636a55"
1201+
integrity sha512-ImAQHxmpMneJ/4S8BRFhjt1MZ3bppmpRPYYNyzeQPeFN288YKbb8TmmISQEbtfkQ1BPASvYZU5doIZOPBAqENQ==
1202+
11581203
"@azure/msal-common@14.7.1":
11591204
version "14.7.1"
11601205
resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.7.1.tgz#b13443fbacc87ce2019a91e81a6582ea73847c75"
@@ -1169,6 +1214,15 @@
11691214
jsonwebtoken "^9.0.0"
11701215
uuid "^8.3.0"
11711216

1217+
"@azure/msal-node@^2.9.2":
1218+
version "2.15.0"
1219+
resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.15.0.tgz#50bf8e692a6656027c073a75d877a8a478aafdfd"
1220+
integrity sha512-gVPW8YLz92ZeCibQH2QUw96odJoiM3k/ZPH3f2HxptozmH6+OnyyvKXo/Egg39HAM230akarQKHf0W74UHlh0Q==
1221+
dependencies:
1222+
"@azure/msal-common" "14.15.0"
1223+
jsonwebtoken "^9.0.0"
1224+
uuid "^8.3.0"
1225+
11721226
"@azure/storage-blob@12.18.x":
11731227
version "12.18.0"
11741228
resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.18.0.tgz#9dd001c9aa5e972216f5af15131009086cfeb59e"

0 commit comments

Comments
 (0)