Merge pull request #888 from bcgov/feature/registration-validation

Upload Stats

Author: saikrishnametpalli-nttdata

server/StrDss.Service/BizLicenceService.cs

@@ -85,8 +85,8 @@ public async Task ProcessBizLicenceUploadMainAsync(DssUploadDelivery upload)
 
         private async Task ProcessBizLicenceUploadSubAsync(DssUploadDelivery upload)
         {
-            var count = 0;
-
+            var processedCount = 0;
+            var errorCount = 0;
             var header = upload.SourceHeaderTxt ?? "";
 
             var linesToProcess = await _uploadRepo.GetUploadLinesToProcessAsync(upload.UploadDeliveryId);
@@ -95,8 +95,6 @@ private async Task ProcessBizLicenceUploadSubAsync(DssUploadDelivery upload)
             foreach (var lineToProcess in linesToProcess)
             {
                 var uploadLine = await _uploadRepo.GetUploadLineAsync(upload.UploadDeliveryId, lineToProcess.OrgCd, lineToProcess.SourceRecordNo);
-
-                count++;
                 var errors = new Dictionary<string, List<string>>();
                 var csvConfig = CsvHelperUtils.GetConfig(errors, false);
 
@@ -105,7 +103,19 @@ private async Task ProcessBizLicenceUploadSubAsync(DssUploadDelivery upload)
                 var csvReader = new CsvReader(textReader, csvConfig);
 
                 var (orgCd, sourceRecordNo) = await ProcessLine(upload, header, uploadLine, csvReader);
-            }
+                processedCount++;
+                if (uploadLine.IsValidationFailure == true || uploadLine.IsSystemFailure == true) errorCount++;
+             }
+
+            upload.UploadStatus = errorCount > 0 ? UploadStatus.Failed : UploadStatus.Processed;
+            upload.RegistrationStatus = UploadStatus.Processed;
+            upload.UploadLinesTotal = lineCount;
+            upload.UploadLinesProcessed = processedCount;
+            upload.UploadLinesSuccess = processedCount - errorCount;
+            upload.UploadLinesError = errorCount;
+            upload.RegistrationLinesSuccess = 0;
+            upload.RegistrationLinesError = 0;
+            _unitOfWork.Commit();
         }
 
         private async Task<(string orgCd, string sourceRecordNo)> ProcessLine(DssUploadDelivery upload, string header, DssUploadLine line, CsvReader csvReader)

server/StrDss.Service/HttpClients/ServiceCollectionExtension.cs

@@ -1,4 +1,4 @@
-using System.Net.Http.Headers;
+using System.Net.Http.Headers;
 using System.Text;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;

server/StrDss.Service/RentalListingReportService.cs

@@ -104,7 +104,7 @@ public async Task ProcessRentalReportUploadAsync(DssUploadDelivery upload)
 
             while (csv.Read())
             {
-                //if (processedCount >= linesPerJob) break; //To process x lines per job
+                if (processedCount >= 500) break; //To process x lines per job
 
                 count++;
                 isLastLine = count == lineCount;
@@ -136,24 +136,29 @@ public async Task ProcessRentalReportUploadAsync(DssUploadDelivery upload)
                 _logger.LogInformation($"Finishing listing ({row.OrgCd} - {row.ListingId}): {stopwatch.Elapsed.TotalMilliseconds} milliseconds - {processedCount}/{lineCount}");
             }
 
+            // Because we may be processing these uploads in blocks of 500, we need to do some running totals and update the upload record with the stats we need.
+            var totalProcessed = upload.UploadLinesProcessed + processedCount;
+            var totalRegErrors = upload.RegistrationLinesError + regErrorCount;
+            var totalErrors = upload.UploadLinesError + errorCount;
+
+            // Update the delivery record with the stats we need.
+            upload.UploadStatus = totalProcessed == lineCount ? UploadStatus.Processed : UploadStatus.Pending; // Current logic has only processed or pending for this upload
+            upload.RegistrationStatus = totalProcessed == lineCount ? totalRegErrors > 0 ? UploadStatus.Failed : UploadStatus.Processed : UploadStatus.Pending;
+            upload.UploadLinesTotal = lineCount;
+            upload.UploadLinesProcessed = totalProcessed;
+            upload.UploadLinesSuccess = totalProcessed - totalErrors;
+            upload.UploadLinesError = totalErrors;
+            upload.RegistrationLinesSuccess = totalProcessed - totalRegErrors;
+            upload.RegistrationLinesError = regErrorCount;
+            _unitOfWork.Commit();
+
             if (!isLastLine)
             {
                 processStopwatch.Stop();
                 _logger.LogInformation($"Processed {processedCount} lines: {report.ReportPeriodYm.ToString("yyyy-MM")}, {report.ProvidingOrganization.OrganizationNm} - {processStopwatch.Elapsed.TotalSeconds} seconds");
                 return;
             }
 
-            // Update the delivery record with the stats we need.
-            upload.UploadStatus = errorCount > 0 ? UploadStatus.Failed : UploadStatus.Processed;
-            upload.RegistrationStatus = regErrorCount > 0 ? UploadStatus.Failed : UploadStatus.Processed;
-            upload.UploadLinesTotal = lineCount;
-            upload.UploadLinesProcessed = processedCount;
-            upload.UploadLinesSuccess = processedCount - errorCount;
-            upload.UploadLinesError = errorCount;
-            upload.RegistrationLinesSuccess = processedCount - regErrorCount;
-            upload.RegistrationLinesError = regErrorCount;
-            _unitOfWork.Commit();
-
             if (errorCount > 0)
             {
                 if (upload.UpdUserGuid == null) return;

server/StrDss.Service/TakedownConfirmationService.cs

@@ -51,7 +51,7 @@ public async Task ProcessTakedownConfirmationUploadAsync(DssUploadDelivery uploa
 
             _logger.LogInformation($"Processing Takedown Confirmation ({upload.UploadDeliveryId}) {upload.ProvidingOrganization.OrganizationNm} - {upload.ReportPeriodYm}");
 
-            var count = 0;
+            var processedCount = 0;
 
             var header = upload.SourceHeaderTxt ?? "";
 
@@ -60,7 +60,6 @@ public async Task ProcessTakedownConfirmationUploadAsync(DssUploadDelivery uploa
 
             foreach (var line in linesToProcess)
             {
-                count++;
                 var errors = new Dictionary<string, List<string>>();
                 var csvConfig = CsvHelperUtils.GetConfig(errors, false);
 
@@ -69,8 +68,19 @@ public async Task ProcessTakedownConfirmationUploadAsync(DssUploadDelivery uploa
                 var csvReader = new CsvReader(textReader, csvConfig);
 
                 var (orgCd, listingId) = await ProcessLine(upload, header, line, csvReader);
+                processedCount++;
             }
 
+            upload.UploadStatus = UploadStatus.Processed;
+            upload.RegistrationStatus = UploadStatus.Processed;
+            upload.UploadLinesTotal = lineCount;
+            upload.UploadLinesProcessed = processedCount;
+            upload.UploadLinesSuccess = processedCount;
+            upload.UploadLinesError = 0;
+            upload.RegistrationLinesSuccess = 0;
+            upload.RegistrationLinesError = 0;
+            _unitOfWork.Commit();
+
             processStopwatch.Stop();
 
             _logger.LogInformation($"Finished Takedown Confirmation: {upload.ReportPeriodYm?.ToString("yyyy-MM")}, {upload.ProvidingOrganization.OrganizationNm} - {processStopwatch.Elapsed.TotalSeconds} seconds");