Skip to content

Commit b91122e

Browse files
author
bhavik
committed
feat: Added support for azure blob-storage in backend
1 parent a5344f5 commit b91122e

File tree

9 files changed

+224
-70
lines changed

9 files changed

+224
-70
lines changed

apps/api/src/.env.development

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

apps/api/src/.env.production

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

apps/api/src/.env.test

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@ RABBITMQ_CONN_URL=amqp://guest:guest@localhost:5672
99
MONGO_URL=mongodb://localhost:27017/impler-db
1010

1111
# Storage
12+
STORAGE_TYPE=s3
1213
S3_LOCAL_STACK=http://localhost:9000
1314
S3_REGION=us-east-1
1415
S3_BUCKET_NAME=impler
1516
AWS_ACCESS_KEY_ID=impler
1617
AWS_SECRET_ACCESS_KEY=implers3cr3t
1718

19+
AZURE_STORAGE_CONNECTION_STRING=
20+
AZURE_STORAGE_CONTAINER=
21+
1822
# Analytics
1923
SENTRY_DSN=
2024

@@ -27,3 +31,6 @@ COOKIE_DOMAIN=
2731
GITHUB_OAUTH_REDIRECT=
2832
GITHUB_OAUTH_CLIENT_ID=
2933
GITHUB_OAUTH_CLIENT_SECRET=
34+
35+
# Code Execution
36+
EXECUTION_MODE=UNSANDBOXED

apps/api/src/.example.env

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@ MONGO_URL=mongodb://localhost:27017/impler-db
1212
SENTRY_DSN=
1313

1414
# Storage
15+
STORAGE_TYPE=s3
1516
S3_LOCAL_STACK=http://localhost:9000
1617
S3_REGION=us-east-1
1718
S3_BUCKET_NAME=impler
1819
AWS_ACCESS_KEY_ID=impler
1920
AWS_SECRET_ACCESS_KEY=implers3cr3t
2021

22+
AZURE_STORAGE_CONNECTION_STRING=
23+
AZURE_STORAGE_CONTAINER=
24+
2125
# URLs
2226
WIDGET_BASE_URL=http://localhost:3500
2327
WEB_BASE_URL=http://localhost:4200
@@ -34,3 +38,6 @@ SES_ACCESS_KEY_ID=
3438
SES_SECRET_ACCESS_KEY=
3539
EMAIL_FROM=
3640
EMAIL_FROM_NAME=
41+
42+
# Code Execution
43+
EXECUTION_MODE=UNSANDBOXED

apps/api/src/app/shared/shared.module.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ import {
3030
SESEmailService,
3131
FileNameService,
3232
NameService,
33+
AzureStorageService,
3334
} from '@impler/services';
35+
import { StorageTypeEnum } from '@impler/shared';
3436

3537
const DAL_MODELS = [
3638
ProjectRepository,
@@ -59,7 +61,7 @@ const UTILITY_SERVICES = [CSVFileService2, FileNameService, NameService, ExcelFi
5961
const dalService = new DalService();
6062

6163
function getStorageServiceClass() {
62-
return S3StorageService;
64+
return process.env.STORAGE_TYPE === StorageTypeEnum.AZURE ? AzureStorageService : S3StorageService;
6365
}
6466

6567
function getEmailServiceClass() {

libs/services/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"@aws-sdk/client-ses": "^3.616.0",
3131
"@aws-sdk/lib-storage": "^3.360.0",
3232
"@aws-sdk/s3-request-presigner": "^3.276.0",
33+
"@azure/storage-blob": "^12.27.0",
3334
"@impler/shared": "workspace:^",
3435
"axios": "1.6.2",
3536
"nodemailer": "^6.9.14",

libs/services/src/storage/storage.service.ts

Lines changed: 152 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,37 @@
1-
import { PassThrough, Readable } from 'stream';
2-
import { Upload } from '@aws-sdk/lib-storage';
1+
import { Readable } from 'stream';
32
import {
43
S3Client,
54
PutObjectCommand,
6-
PutObjectCommandOutput,
75
GetObjectCommand,
86
DeleteObjectCommand,
97
ListBucketsCommand,
108
} from '@aws-sdk/client-s3';
119
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
1210
import { FileNotExistError, Defaults } from '@impler/shared';
1311

12+
// Azure Storage imports
13+
import {
14+
BlobSASPermissions,
15+
BlobServiceClient,
16+
BlockBlobUploadResponse,
17+
ContainerSASPermissions,
18+
} from '@azure/storage-blob';
19+
1420
export interface IFilePath {
1521
path: string;
1622
name: string;
1723
}
1824

25+
export interface StorageResponse {
26+
success: boolean;
27+
metadata?: any;
28+
}
29+
1930
export abstract class StorageService {
20-
abstract uploadFile(
21-
key: string,
22-
file: Buffer | string | PassThrough,
23-
contentType: string
24-
): Promise<PutObjectCommandOutput>;
31+
abstract uploadFile(key: string, file: Buffer | string | Readable, contentType: string): Promise<StorageResponse>;
2532
abstract getFileContent(key: string, encoding?: BufferEncoding): Promise<string>;
2633
abstract getFileStream(key: string): Promise<Readable>;
27-
abstract writeStream(key: string, stream: Readable, contentType: string): Upload;
34+
abstract writeStream(key: string, stream: Readable, contentType: string): Promise<void>;
2835
abstract deleteFile(key: string): Promise<void>;
2936
abstract isConnected(): boolean;
3037
abstract getSignedUrl(key: string): Promise<string>;
@@ -63,15 +70,22 @@ export class S3StorageService implements StorageService {
6370
});
6471
}
6572

66-
async uploadFile(key: string, file: Buffer, contentType: string): Promise<PutObjectCommandOutput> {
73+
async uploadFile(key: string, file: Buffer | string | Readable, contentType: string): Promise<StorageResponse> {
6774
const command = new PutObjectCommand({
6875
Bucket: process.env.S3_BUCKET_NAME,
6976
Key: key,
7077
Body: file,
7178
ContentType: contentType,
7279
});
7380

74-
return await this.s3.send(command);
81+
const result = await this.s3.send(command);
82+
return {
83+
success: true,
84+
metadata: {
85+
eTag: result.ETag,
86+
versionId: result.VersionId,
87+
},
88+
};
7589
}
7690

7791
async getFileContent(key: string, encoding = 'utf8' as BufferEncoding): Promise<string> {
@@ -81,7 +95,6 @@ export class S3StorageService implements StorageService {
8195
Key: key,
8296
});
8397
const data = await this.s3.send(command);
84-
8598
return await streamToString(data.Body as Readable, encoding);
8699
} catch (error) {
87100
if (error.code === Defaults.NOT_FOUND_STATUS_CODE || error.message === 'The specified key does not exist.') {
@@ -98,7 +111,6 @@ export class S3StorageService implements StorageService {
98111
Key: key,
99112
});
100113
const data = await this.s3.send(command);
101-
102114
return data.Body as Readable;
103115
} catch (error) {
104116
if (error.code === Defaults.NOT_FOUND_STATUS_CODE || error.message === 'The specified key does not exist.') {
@@ -108,17 +120,14 @@ export class S3StorageService implements StorageService {
108120
}
109121
}
110122

111-
writeStream(key: string, stream: Readable | PassThrough, contentType: string): Upload {
112-
return new Upload({
113-
client: this.s3,
114-
queueSize: 4,
115-
params: {
116-
Bucket: process.env.S3_BUCKET_NAME,
117-
Key: key,
118-
Body: stream,
119-
ContentType: contentType,
120-
},
123+
async writeStream(key: string, stream: Readable, contentType: string): Promise<void> {
124+
const command = new PutObjectCommand({
125+
Bucket: process.env.S3_BUCKET_NAME,
126+
Key: key,
127+
Body: stream,
128+
ContentType: contentType,
121129
});
130+
await this.s3.send(command);
122131
}
123132

124133
async deleteFile(key: string): Promise<void> {
@@ -134,16 +143,126 @@ export class S3StorageService implements StorageService {
134143
}
135144

136145
async getSignedUrl(key: string): Promise<string> {
137-
return await getSignedUrl(
138-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
139-
// @ts-ignore
140-
this.s3,
141-
new GetObjectCommand({
142-
Bucket: process.env.S3_BUCKET_NAME,
143-
Key: key,
144-
}),
145-
// eslint-disable-next-line no-magic-numbers
146-
{ expiresIn: 15 * 60 } // 15 minutes
146+
const command = new GetObjectCommand({
147+
Bucket: process.env.S3_BUCKET_NAME,
148+
Key: key,
149+
});
150+
return getSignedUrl(this.s3, command, { expiresIn: 3600 });
151+
}
152+
}
153+
154+
export class AzureStorageService implements StorageService {
155+
private isAzureConnected = false;
156+
private blobServiceClient: BlobServiceClient;
157+
private containerClient: any;
158+
159+
constructor() {
160+
if (!process.env.AZURE_STORAGE_CONNECTION_STRING) {
161+
throw new Error('AZURE_STORAGE_CONNECTION_STRING is not configured');
162+
}
163+
164+
this.blobServiceClient = BlobServiceClient.fromConnectionString(process.env.AZURE_STORAGE_CONNECTION_STRING);
165+
this.containerClient = this.blobServiceClient.getContainerClient(
166+
process.env.AZURE_STORAGE_CONTAINER || 'default-container'
147167
);
168+
169+
// Verify connection
170+
this.blobServiceClient
171+
.listContainers()
172+
.next()
173+
.then(() => {
174+
this.isAzureConnected = true;
175+
})
176+
.catch(() => {
177+
this.isAzureConnected = false;
178+
});
179+
}
180+
181+
async uploadFile(key: string, file: Buffer | string | Readable, contentType: string): Promise<StorageResponse> {
182+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
183+
let uploadResponse: BlockBlobUploadResponse;
184+
185+
if (typeof file === 'string') {
186+
const buffer = Buffer.from(file);
187+
uploadResponse = await blockBlobClient.upload(buffer, buffer.length);
188+
} else if (file instanceof Buffer) {
189+
uploadResponse = await blockBlobClient.upload(file, file.length);
190+
} else {
191+
// For Readable streams
192+
const chunks: Buffer[] = [];
193+
for await (const chunk of file) {
194+
chunks.push(chunk);
195+
}
196+
const buffer = Buffer.concat(chunks);
197+
uploadResponse = await blockBlobClient.upload(buffer, buffer.length);
198+
}
199+
200+
await blockBlobClient.setHTTPHeaders({ blobContentType: contentType });
201+
return {
202+
success: true,
203+
metadata: {
204+
eTag: uploadResponse.etag,
205+
},
206+
};
207+
}
208+
209+
async getFileContent(key: string, encoding = 'utf8' as BufferEncoding): Promise<string> {
210+
try {
211+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
212+
const downloadResponse = await blockBlobClient.download();
213+
const content = await downloadResponse.blobBody?.toString(encoding);
214+
if (!content) {
215+
throw new Error('Failed to download file content');
216+
}
217+
return content;
218+
} catch (error) {
219+
if (error.name === 'RestError' && error.statusCode === 404) {
220+
throw new FileNotExistError(key);
221+
}
222+
throw error;
223+
}
224+
}
225+
226+
async getFileStream(key: string): Promise<Readable> {
227+
try {
228+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
229+
const downloadResponse = await blockBlobClient.download();
230+
return Readable.from(downloadResponse.blobBody as any);
231+
} catch (error) {
232+
if (error.name === 'RestError' && error.statusCode === 404) {
233+
throw new FileNotExistError(key);
234+
}
235+
throw error;
236+
}
237+
}
238+
239+
async writeStream(key: string, stream: Readable, contentType: string): Promise<void> {
240+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
241+
const chunks: Buffer[] = [];
242+
for await (const chunk of stream) {
243+
chunks.push(chunk);
244+
}
245+
const buffer = Buffer.concat(chunks);
246+
await blockBlobClient.upload(buffer, buffer.length);
247+
await blockBlobClient.setHTTPHeaders({ blobContentType: contentType });
248+
}
249+
250+
async deleteFile(key: string): Promise<void> {
251+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
252+
await blockBlobClient.delete();
253+
}
254+
255+
isConnected(): boolean {
256+
return this.isAzureConnected;
257+
}
258+
259+
async getSignedUrl(key: string): Promise<string> {
260+
const blockBlobClient = this.containerClient.getBlockBlobClient(key);
261+
const permissions = BlobSASPermissions.parse('r');
262+
263+
return blockBlobClient.generateSasUrl({
264+
permissions,
265+
expiresOn: new Date(Date.now() + 3600 * 1000), // 1 hour expiry
266+
});
148267
}
149268
}

libs/shared/src/config/api.config.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,8 @@ export const ENVTypesEnum = {
77
CI: 'ci',
88
LOCAL: 'local',
99
};
10+
11+
export const StorageTypeEnum = {
12+
S3: 's3',
13+
AZURE: 'azure',
14+
};

0 commit comments

Comments
 (0)