Skip to content

Commit 73847f2

Browse files
chavda-bhavikchavda-bhavik
authored
Azure blob storage support (#989)
2 parents c9f8c14 + bcc845a commit 73847f2

File tree

17 files changed

+415
-45
lines changed

17 files changed

+415
-45
lines changed

‎apps/api/src/.env.development

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

‎apps/api/src/.env.production

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

‎apps/api/src/.env.test

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

‎apps/api/src/.example.env

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@ MONGO_URL=mongodb://localhost:27017/impler-db
1212
SENTRY_DSN=
1313

1414
# Storage
15+
STORAGE_TYPE=s3
1516
S3_LOCAL_STACK=http://localhost:9000
1617
S3_REGION=us-east-1
1718
S3_BUCKET_NAME=impler
1819
AWS_ACCESS_KEY_ID=impler
1920
AWS_SECRET_ACCESS_KEY=implers3cr3t
2021

22+
AZURE_STORAGE_CONNECTION_STRING=
23+
AZURE_STORAGE_CONTAINER=
24+
2125
# URLs
2226
WIDGET_BASE_URL=http://localhost:3500
2327
WEB_BASE_URL=http://localhost:4200
@@ -34,3 +38,6 @@ SES_ACCESS_KEY_ID=
3438
SES_SECRET_ACCESS_KEY=
3539
EMAIL_FROM=
3640
EMAIL_FROM_NAME=
41+
42+
# Code Execution
43+
EXECUTION_MODE=UNSANDBOXED

‎apps/api/src/app/shared/shared.module.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import {
2222
FailedWebhookRetryRequestsRepository,
2323
WebhookLogRepository,
2424
} from '@impler/dal';
25+
import { StorageTypeEnum } from '@impler/shared';
2526
import { CSVFileService2, ExcelFileService } from './services/file/file.service';
2627
import {
2728
S3StorageService,
@@ -30,6 +31,7 @@ import {
3031
SESEmailService,
3132
FileNameService,
3233
NameService,
34+
AzureStorageService,
3335
} from '@impler/services';
3436
import { WebSocketService } from './services';
3537

@@ -60,7 +62,7 @@ const UTILITY_SERVICES = [CSVFileService2, FileNameService, NameService, ExcelFi
6062
const dalService = new DalService();
6163

6264
function getStorageServiceClass() {
63-
return S3StorageService;
65+
return process.env.STORAGE_TYPE === StorageTypeEnum.AZURE ? AzureStorageService : S3StorageService;
6466
}
6567

6668
function getEmailServiceClass() {

‎apps/queue-manager/src/.env.development

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,23 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
33
MONGO_URL=mongodb://localhost:27017/impler-db
44

55
# Storage
6+
STORAGE_TYPE=s3
67
S3_REGION=us-east-1
78
S3_LOCAL_STACK=http://localhost:4566
89
S3_BUCKET_NAME=impler
910
AWS_ACCESS_KEY_ID=
1011
AWS_SECRET_ACCESS_KEY=
1112

13+
AZURE_STORAGE_CONNECTION_STRING=
14+
AZURE_STORAGE_CONTAINER=
15+
1216
# Analytics
13-
SENTRY_DSN=
17+
SENTRY_DSN=
18+
19+
# Email
20+
SES_REGION=
21+
SES_ACCESS_KEY_ID=
22+
SES_SECRET_ACCESS_KEY=
23+
EMAIL_FROM=
24+
ALERT_EMAIL_FROM=
25+
EMAIL_FROM_NAME=

‎apps/queue-manager/src/.env.production

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,23 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
33
MONGO_URL=mongodb://localhost:27017/impler-db
44

55
# Storage
6+
STORAGE_TYPE=s3
67
S3_REGION=us-east-1
78
S3_LOCAL_STACK=http://localhost:4566
89
S3_BUCKET_NAME=impler
910
AWS_ACCESS_KEY_ID=
1011
AWS_SECRET_ACCESS_KEY=
1112

13+
AZURE_STORAGE_CONNECTION_STRING=
14+
AZURE_STORAGE_CONTAINER=
15+
1216
# Analytics
13-
SENTRY_DSN=
17+
SENTRY_DSN=
18+
19+
# Email
20+
SES_REGION=
21+
SES_ACCESS_KEY_ID=
22+
SES_SECRET_ACCESS_KEY=
23+
EMAIL_FROM=
24+
ALERT_EMAIL_FROM=
25+
EMAIL_FROM_NAME=

‎apps/queue-manager/src/.env.test

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,20 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
33
MONGO_URL=mongodb://localhost:27017/impler-db
44

55
# Storage
6+
STORAGE_TYPE=s3
67
S3_REGION=us-east-1
78
S3_LOCAL_STACK=http://localhost:4566
89
S3_BUCKET_NAME=impler
910
AWS_ACCESS_KEY_ID=
1011
AWS_SECRET_ACCESS_KEY=
1112

1213
# Analytics
13-
SENTRY_DSN=
14+
SENTRY_DSN=
15+
16+
# Email
17+
SES_REGION=
18+
SES_ACCESS_KEY_ID=
19+
SES_SECRET_ACCESS_KEY=
20+
EMAIL_FROM=
21+
ALERT_EMAIL_FROM=
22+
EMAIL_FROM_NAME=

‎apps/queue-manager/src/.example.env

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,18 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
44
MONGO_URL=mongodb://localhost:27017/impler-db
55

66
# Storage
7+
STORAGE_TYPE=s3
78
S3_REGION=us-east-1
89
S3_LOCAL_STACK=http://localhost:4566
910
S3_BUCKET_NAME=impler
1011

1112
# Analytics
12-
SENTRY_DSN=
13+
SENTRY_DSN=
14+
15+
# Email
16+
SES_REGION=
17+
SES_ACCESS_KEY_ID=
18+
SES_SECRET_ACCESS_KEY=
19+
EMAIL_FROM=
20+
ALERT_EMAIL_FROM=
21+
EMAIL_FROM_NAME=

‎apps/queue-manager/src/config/env-validator.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,17 @@
11
import { str, ValidatorSpec } from 'envalid';
22
import * as envalid from 'envalid';
3-
import { ENVTypesEnum } from '@impler/shared';
3+
import { ENVTypesEnum, StorageTypeEnum } from '@impler/shared';
44

55
// eslint-disable-next-line @typescript-eslint/no-explicit-any
66
const validators: { [K in keyof any]: ValidatorSpec<any[K]> } = {
77
NODE_ENV: str({
88
choices: [ENVTypesEnum.DEV, ENVTypesEnum.TEST, ENVTypesEnum.PROD, ENVTypesEnum.CI, ENVTypesEnum.LOCAL],
99
default: ENVTypesEnum.LOCAL,
1010
}),
11+
STORAGE_TYPE: str({
12+
choices: [StorageTypeEnum.S3, StorageTypeEnum.AZURE],
13+
default: StorageTypeEnum.S3,
14+
}),
1115
MONGO_URL: str(),
1216
RABBITMQ_CONN_URL: str(),
1317
S3_LOCAL_STACK: str({

‎apps/queue-manager/src/helpers/services.helper.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
1-
import { EmailService, S3StorageService, SESEmailService, StorageService } from '@impler/services';
1+
import { StorageTypeEnum } from '@impler/shared';
2+
import { AzureStorageService, EmailService, S3StorageService, SESEmailService, StorageService } from '@impler/services';
23

34
let storageService: StorageService;
45
let emailService: EmailService;
56

67
// Implementing singleton pattern for storage service
78
export function getStorageServiceClass() {
89
if (storageService) return storageService;
9-
storageService = new S3StorageService();
10+
storageService =
11+
process.env.STORAGE_TYPE === StorageTypeEnum.AZURE ? new AzureStorageService() : new S3StorageService();
1012

1113
return storageService;
1214
}

‎docker/.env.example

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,16 @@ MONGO_URL=mongodb://mongodb:27017/impler
77
RABBITMQ_CONN_URL=amqp://guest:guest@rabbitmq:5672
88

99
# AWS
10+
STORAGE_TYPE=s3
1011
S3_LOCAL_STACK=http://localhost:9000
1112
S3_BUCKET_NAME=impler
1213
S3_REGION=us-east-1
1314
AWS_ACCESS_KEY_ID=impler
1415
AWS_SECRET_ACCESS_KEY=implers3cr3t
1516

17+
AZURE_STORAGE_CONNECTION_STRING=
18+
AZURE_STORAGE_CONTAINER=
19+
1620
# Ports
1721
API_PORT=3000
1822

‎docker/docker-compose.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,12 @@ services:
3030
WIDGET_BASE_URL: ${WIDGET_BASE_URL}
3131
WEB_BASE_URL: ${WEB_BASE_URL}
3232
MONGO_URL: ${MONGO_URL}
33+
STORAGE_TYPE: ${STORAGE_TYPE}
3334
S3_REGION: ${S3_REGION}
3435
S3_LOCAL_STACK: ${S3_LOCAL_STACK}
3536
S3_BUCKET_NAME: ${S3_BUCKET_NAME}
37+
AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
38+
AZURE_STORAGE_CONTAINER: ${AZURE_STORAGE_CONTAINER}
3639
JWT_SECRET: ${JWT_SECRET}
3740
COOKIE_DOMAIN: ${COOKIE_DOMAIN}
3841
RABBITMQ_CONN_URL: ${RABBITMQ_CONN_URL}
@@ -62,10 +65,13 @@ services:
6265
NODE_ENV: ${NODE_ENV}
6366
MONGO_URL: ${MONGO_URL}
6467
RABBITMQ_CONN_URL: ${RABBITMQ_CONN_URL}
65-
S3_REGION: ${S3_REGION}
6668
API_ROOT_URL: ${API_ROOT_URL}
69+
STORAGE_TYPE: ${STORAGE_TYPE}
70+
S3_REGION: ${S3_REGION}
6771
S3_LOCAL_STACK: ${S3_LOCAL_STACK}
6872
S3_BUCKET_NAME: ${S3_BUCKET_NAME}
73+
AZURE_STORAGE_CONNECTION_STRING: ${AZURE_STORAGE_CONNECTION_STRING}
74+
AZURE_STORAGE_CONTAINER: ${AZURE_STORAGE_CONTAINER}
6975
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
7076
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
7177
SES_REGION: ${SES_REGION}

‎libs/services/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
"@aws-sdk/client-ses": "^3.616.0",
3232
"@aws-sdk/lib-storage": "^3.360.0",
3333
"@aws-sdk/s3-request-presigner": "^3.276.0",
34+
"@azure/storage-blob": "^12.27.0",
3435
"@impler/shared": "workspace:^",
3536
"axios": "1.6.2",
3637
"nodemailer": "^6.9.14",

0 commit comments

Comments
 (0)