Feat/max records

apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts

@@ -25,7 +25,6 @@ export class CreateUserJob {
 
     try {
       const mimeType = await getMimeType(url);
-      console.log('mime type is >>', mimeType);
 
       if (isValidXMLMimeType(mimeType)) {
         const abortController = new AbortController();

apps/api/src/app/review/review.controller.ts

@@ -86,6 +86,7 @@ export class ReviewController {
     const uploadData = await this.getUpload.execute({
       uploadId: _uploadId,
     });
+
     if (!uploadData) throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
 
     return await this.getFileInvalidData.execute(
@@ -123,7 +124,7 @@ export class ReviewController {
   @ApiOperation({
     summary: 'Confirm review data for uploaded file',
   })
-  async doConfirmReview(@Param('uploadId', ValidateMongoId) _uploadId: string) {
+  async doConfirmReview(@Param('uploadId', ValidateMongoId) _uploadId: string, @Body() body: { maxRecords?: number }) {
     const uploadInformation = await this.getUpload.execute({
       uploadId: _uploadId,
       select: 'status _validDataFileId _invalidDataFileId totalRecords invalidRecords _templateId',
@@ -135,7 +136,7 @@ export class ReviewController {
     // upload files with status reviewing can only be confirmed
     validateUploadStatus(uploadInformation.status as UploadStatusEnum, [UploadStatusEnum.REVIEWING]);
 
-    return this.startProcess.execute(_uploadId);
+    return this.startProcess.execute(_uploadId, body.maxRecords);
   }
 
   @Put(':uploadId/record')
@@ -168,7 +169,7 @@ export class ReviewController {
     @Body() { indexes, valid, invalid }: DeleteRecordsDto,
     @Param('uploadId', ValidateMongoId) _uploadId: string
   ) {
-    await this.deleteRecord.execute(_uploadId, indexes, valid, invalid);
+    return await this.deleteRecord.execute(_uploadId, indexes, valid, invalid);
   }
 
   @Put(':uploadId/replace')

apps/api/src/app/review/usecases/delete-record/delete-record.usecase.ts

@@ -8,21 +8,61 @@ export class DeleteRecord {
     private uploadRepository: UploadRepository
   ) {}
 
-  async execute(_uploadId: string, indexes: number[], valid: number, invalid: number) {
-    await this.dalService.deleteRecords(_uploadId, indexes);
-    if (typeof valid !== 'undefined' && typeof invalid !== 'undefined') {
-      await this.uploadRepository.update(
-        {
-          _id: _uploadId,
-        },
-        {
-          $inc: {
-            totalRecords: -indexes.length,
-            validRecords: -valid,
-            invalidRecords: -invalid,
+  async execute(
+    uploadId: string,
+    recordIndexesToDelete: number[],
+    validRecordsToDelete: number,
+    invalidRecordsToDelete: number
+  ) {
+    await this.dalService.deleteRecords(uploadId, recordIndexesToDelete);
+
+    if (typeof validRecordsToDelete !== 'undefined' && typeof invalidRecordsToDelete !== 'undefined') {
+      // Get current upload statistics to prevent negative values
+      const currentUploadData = await this.uploadRepository.findOne({ _id: uploadId });
+
+      if (currentUploadData) {
+        // Calculate safe decrement values to prevent database corruption
+        const safeValidRecordsDecrement = Math.min(validRecordsToDelete, currentUploadData.validRecords || 0);
+        const safeInvalidRecordsDecrement = Math.min(invalidRecordsToDelete, currentUploadData.invalidRecords || 0);
+        const safeTotalRecordsDecrement = Math.min(recordIndexesToDelete.length, currentUploadData.totalRecords || 0);
+
+        await this.uploadRepository.update(
+          {
+            _id: uploadId,
           },
+          {
+            $inc: {
+              totalRecords: -safeTotalRecordsDecrement,
+              validRecords: -safeValidRecordsDecrement,
+              invalidRecords: -safeInvalidRecordsDecrement,
+            },
+          }
+        );
+
+        // Double-check and ensure no negative values exist in database
+        const updatedUploadData = await this.uploadRepository.findOne({ _id: uploadId });
+        if (updatedUploadData) {
+          const fieldsToCorrect: any = {};
+          let requiresCorrection = false;
+
+          if ((updatedUploadData.totalRecords || 0) < 0) {
+            fieldsToCorrect.totalRecords = 0;
+            requiresCorrection = true;
+          }
+          if ((updatedUploadData.validRecords || 0) < 0) {
+            fieldsToCorrect.validRecords = 0;
+            requiresCorrection = true;
+          }
+          if ((updatedUploadData.invalidRecords || 0) < 0) {
+            fieldsToCorrect.invalidRecords = 0;
+            requiresCorrection = true;
+          }
+
+          if (requiresCorrection) {
+            await this.uploadRepository.update({ _id: uploadId }, { $set: fieldsToCorrect });
+          }
         }
-      );
+      }
     }
   }
 }

apps/api/src/app/review/usecases/get-upload-data/get-upload-data.usecase.ts

@@ -16,7 +16,7 @@ export class GetUploadData {
     const data = await this.dalService.getRecords(_uploadId, page, limit, type);
 
     return {
-      data,
+      data: data.sort((a, b) => a.index - b.index),
       limit,
       page,
       totalRecords,

apps/api/src/app/review/usecases/start-process/start-process.usecase.ts

@@ -12,6 +12,7 @@ import { PaymentAPIService } from '@impler/services';
 import { QueueService } from '@shared/services/queue.service';
 import { AmplitudeService } from '@shared/services/amplitude.service';
 import { DalService, TemplateEntity, TemplateRepository, UploadEntity, UploadRepository } from '@impler/dal';
+import { MaxRecordsExceededException } from '@shared/exceptions/max-records.exception';
 
 @Injectable()
 export class StartProcess {
@@ -24,7 +25,7 @@ export class StartProcess {
     private templateRepository: TemplateRepository
   ) {}
 
-  async execute(_uploadId: string) {
+  async execute(_uploadId: string, maxRecords?: number) {
     let uploadInfo = await this.uploadRepository.getUploadWithTemplate(_uploadId, ['destination']);
     let importedData;
     const destination = (uploadInfo._templateId as unknown as TemplateEntity)?.destination;
@@ -44,7 +45,7 @@ export class StartProcess {
       });
     }
 
-    await this.updateTemplateStatistics({ uploadInfo, userEmail });
+    await this.updateTemplateStatistics({ uploadInfo, userEmail, maxRecords });
 
     // if destination is frontend or not defined then complete the upload process
     if (
@@ -117,32 +118,59 @@ export class StartProcess {
     return importedData;
   }
 
-  private async updateTemplateStatistics({ uploadInfo, userEmail }: { uploadInfo: UploadEntity; userEmail: string }) {
-    //if its a file based import do-review will handle the further process
+  private async updateTemplateStatistics({
+    uploadInfo,
+    userEmail,
+    maxRecords,
+  }: {
+    uploadInfo: UploadEntity;
+    userEmail: string;
+    maxRecords?: number;
+  }) {
+    // If it's a file based import do-review will handle the further process
     if (uploadInfo._uploadedFileId || uploadInfo.originalFileName) {
       return;
     }
-    await this.templateRepository.findOneAndUpdate(
-      {
-        _id: uploadInfo._templateId,
-      },
-      {
-        $inc: {
-          totalUploads: uploadInfo.totalRecords,
-          totalRecords: uploadInfo.totalRecords,
-          totalInvalidRecords: uploadInfo.invalidRecords,
+
+    // Check max records limit BEFORE updating statistics
+    if (maxRecords && uploadInfo.validRecords > maxRecords) {
+      throw new MaxRecordsExceededException({
+        maxAllowed: maxRecords,
+      });
+    }
+
+    // Validate that we're not updating with negative values
+    const recordsToAdd = Math.max(0, uploadInfo.totalRecords || 0);
+    const invalidRecordsToAdd = Math.max(0, uploadInfo.invalidRecords || 0);
+    const validRecordsToAdd = Math.max(0, uploadInfo.validRecords || 0);
+
+    // Only update if we have positive values to add
+    if (validRecordsToAdd > 0) {
+      await this.templateRepository.findOneAndUpdate(
+        {
+          _id: uploadInfo._templateId,
         },
-      }
-    );
-
-    await this.paymentAPIService.createEvent(
-      {
-        uploadId: uploadInfo._id,
-        totalRecords: uploadInfo.totalRecords,
-        validRecords: uploadInfo.validRecords,
-        invalidRecords: uploadInfo.invalidRecords,
-      },
-      userEmail
-    );
+        {
+          $inc: {
+            totalUploads: uploadInfo.totalRecords,
+            totalRecords: uploadInfo.totalRecords,
+            totalInvalidRecords: Math.max(0, uploadInfo.invalidRecords),
+          },
+        }
+      );
+    }
+
+    // Only create payment event if we have valid records
+    if (validRecordsToAdd > 0 || recordsToAdd > 0) {
+      await this.paymentAPIService.createEvent(
+        {
+          uploadId: uploadInfo._id,
+          totalRecords: recordsToAdd,
+          validRecords: validRecordsToAdd,
+          invalidRecords: invalidRecordsToAdd,
+        },
+        userEmail
+      );
+    }
   }
 }

apps/api/src/app/shared/exceptions/max-records.exception.ts

@@ -0,0 +1,8 @@
+import { BadRequestException } from '@nestjs/common';
+import { numberFormatter } from '@impler/shared';
+
+export class MaxRecordsExceededException extends BadRequestException {
+  constructor({ maxAllowed, customMessage = null }: { maxAllowed: number; customMessage?: string }) {
+    super(customMessage || `You can nor import records more than ${numberFormatter(maxAllowed)} records.`);
+  }
+}

apps/api/src/app/shared/usecases/get-upload/get-upload.command.ts

@@ -1,5 +1,5 @@
 export class GetUploadCommand {
   uploadId: string;
-
+  maxRecords?: number;
   select?: string;
 }

apps/api/src/app/upload/dtos/upload-request.dto.ts

@@ -1,5 +1,5 @@
 import { ApiProperty } from '@nestjs/swagger';
-import { IsJSON, IsMongoId, IsOptional, IsString } from 'class-validator';
+import { IsJSON, IsMongoId, IsNumberString, IsOptional, IsString } from 'class-validator';
 
 export class UploadRequestDto {
   @ApiProperty({
@@ -61,4 +61,12 @@ export class UploadRequestDto {
   @IsOptional()
   @IsJSON()
   imageSchema?: string;
+
+  @ApiProperty({
+    description: 'Max Number of records to import',
+    required: false,
+  })
+  @IsOptional()
+  @IsNumberString()
+  maxRecords?: string;
 }

apps/api/src/app/upload/upload.controller.ts

@@ -78,14 +78,14 @@ export class UploadController {
     @UploadedFile('file', ValidImportFile) file: Express.Multer.File
   ) {
     return this.makeUploadEntry.execute({
-      file: file,
+      file,
       templateId,
       extra: body.extra,
       schema: body.schema,
       output: body.output,
       importId: body.importId,
       imageSchema: body.imageSchema,
-
+      maxRecords: parseInt(body.maxRecords),
       authHeaderValue: body.authHeaderValue,
       selectedSheetName: body.selectedSheetName,
     });

apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.command.ts

@@ -8,4 +8,5 @@ export class MakeUploadEntryCommand {
   imageSchema?: string;
   authHeaderValue?: string;
   selectedSheetName?: string;
+  maxRecords?: number;
 }

apps/api/src/app/upload/usecases/make-upload-entry/make-upload-entry.usecase.ts

@@ -27,6 +27,7 @@ import { MakeUploadEntryCommand } from './make-upload-entry.command';
 import { FileParseException } from '@shared/exceptions/file-parse-issue.exception';
 import { CSVFileService2, ExcelFileService } from '@shared/services/file';
 import { FileSizeException } from '@shared/exceptions/file-size-limit.exception';
+import { MaxRecordsExceededException } from '@shared/exceptions/max-records.exception';
 
 @Injectable()
 export class MakeUploadEntry {
@@ -52,6 +53,7 @@ export class MakeUploadEntry {
     importId,
     imageSchema,
     selectedSheetName,
+    maxRecords,
   }: MakeUploadEntryCommand) {
     const csvFileService = new CSVFileService2();
     let fileOriginalName: string | undefined, csvFile: string | Express.Multer.File | undefined;
@@ -66,17 +68,33 @@ export class MakeUploadEntry {
         try {
           const fileService = new ExcelFileService();
           const opts = await fileService.getExcelRowsColumnsCount(file, selectedSheetName);
-          this.analyzeLargeFile(opts, true);
+
+          // Check maxRecords restriction for Excel files
+          if (maxRecords !== undefined && maxRecords !== null && opts.rows > maxRecords) {
+            throw new MaxRecordsExceededException({
+              maxAllowed: maxRecords,
+            });
+          }
+
+          this.analyzeLargeFile(opts, true, maxRecords);
           csvFile = await fileService.convertToCsv(file, selectedSheetName);
         } catch (error) {
-          if (error instanceof FileSizeException) {
+          if (error instanceof FileSizeException || error instanceof MaxRecordsExceededException) {
             throw error;
           }
           throw new FileParseException();
         }
       } else if (file.mimetype === FileMimeTypesEnum.CSV) {
         const opts = await csvFileService.getCSVMetaInfo(file);
-        this.analyzeLargeFile(opts, false);
+
+        // Check maxRecords restriction for CSV files
+        if (maxRecords !== undefined && maxRecords !== null && opts.rows > maxRecords) {
+          throw new MaxRecordsExceededException({
+            maxAllowed: maxRecords,
+          });
+        }
+
+        this.analyzeLargeFile(opts, false, maxRecords);
         csvFile = file;
       } else {
         throw new Error('Invalid file type');
@@ -188,15 +206,34 @@ export class MakeUploadEntry {
       originalFileType: file?.mimetype,
     });
   }
+
   roundToNiceNumber(num: number) {
     const niceNumbers = [500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000];
 
     return niceNumbers.reduce((prev, curr) => (Math.abs(curr - num) < Math.abs(prev - num) ? curr : prev));
   }
-  analyzeLargeFile(fileInfo: { rows: number; columns: number }, isExcel?: boolean, maxDataPoints = 5000000) {
+
+  analyzeLargeFile(
+    fileInfo: { rows: number; columns: number },
+    isExcel?: boolean,
+    maxRecords?: number,
+    maxDataPoints = 5000000
+  ) {
     const { columns, rows } = fileInfo;
     const dataPoints = columns * rows;
 
+    // If maxRecords is specified, use it as the limit instead of maxDataPoints calculation
+    if (maxRecords !== undefined && maxRecords !== null) {
+      if (rows > maxRecords) {
+        throw new MaxRecordsExceededException({
+          maxAllowed: maxRecords,
+        });
+      }
+
+      // If maxRecords is specified and file is within limit, skip the dataPoints check
+      return;
+    }
+
     if (dataPoints > maxDataPoints) {
       let suggestedChunkSize = Math.floor(maxDataPoints / columns);
       suggestedChunkSize = this.roundToNiceNumber(suggestedChunkSize);

apps/widget/src/components/Common/Container/Container.tsx

@@ -64,6 +64,7 @@ export function Container({ children }: PropsWithChildren<{}>) {
         texts: deepMerge(WIDGET_TEXTS, data.value.texts),
         config: data.value.config,
         appearance: data.value.appearance,
+        maxRecords: data.value.maxRecords,
         schema:
           typeof data.value.schema === 'string'
             ? data.value.schema
@@ -842,6 +843,7 @@ export function Container({ children }: PropsWithChildren<{}>) {
           templateId={secondaryPayload.templateId}
           authHeaderValue={secondaryPayload?.authHeaderValue}
           sampleFile={secondaryPayload?.sampleFile}
+          maxRecords={secondaryPayload.maxRecords}
           primaryColor={secondaryPayload.primaryColor ?? secondaryPayload.appearance?.primaryColor ?? primaryColor}
         >
           {children}