From 27e3dae79b4826873e0666302b7c0c07bf0bc469 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Wed, 30 Apr 2025 13:44:08 -0700 Subject: [PATCH 01/13] test for only one org add more info add another remove condition 1 concurrent add inner exception log more log another change test with time infinite timeout cancellation token infinite token add more refine test working code --- src/Spd.Manager.ScheduleJob/Contract.cs | 2 +- .../ScheduleJobManager.cs | 5 +- .../Controllers/ScheduleJobController.cs | 3 +- .../appsettings.json | 1 + .../Org/Contract.cs | 2 +- .../Org/OrgRepository.cs | 72 +++++++++++++++++-- .../ScheduleJobSession/Mappings.cs | 2 +- .../ScheduleJobSessionRepository.cs | 10 ++- src/Spd.Utilities.Dynamics/Configurer.cs | 5 +- 9 files changed, 85 insertions(+), 17 deletions(-) diff --git a/src/Spd.Manager.ScheduleJob/Contract.cs b/src/Spd.Manager.ScheduleJob/Contract.cs index 8f9591669e..e8d6be304f 100644 --- a/src/Spd.Manager.ScheduleJob/Contract.cs +++ b/src/Spd.Manager.ScheduleJob/Contract.cs @@ -8,6 +8,6 @@ public interface IScheduleJobManager } #region run schedule job session - public record RunScheduleJobSessionCommand(Guid JobSessionId) : IRequest; + public record RunScheduleJobSessionCommand(Guid JobSessionId, int ConcurrentRequests) : IRequest; #endregion } diff --git a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs index 953d24c59c..e1e72ac38e 100644 --- a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs +++ b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs @@ -42,15 +42,16 @@ public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationTok //used for Org MonthlyInvoice if (resp.PrimaryEntity == "account" && resp.EndPoint.Equals("spd_MonthlyInvoice")) { + using var cts = new CancellationTokenSource(); // no timeout try { Stopwatch stopwatch = Stopwatch.StartNew(); - var result = await _orgRepository.RunMonthlyInvoiceAsync(ct); + var result = await _orgRepository.RunMonthlyInvoiceAsync(cmd.ConcurrentRequests, cts.Token); stopwatch.Stop(); //update result in JobSession UpdateScheduleJobSessionCmd updateResultCmd = CreateUpdateScheduleJobSessionCmd(cmd.JobSessionId, result, Decimal.Round((decimal)(stopwatch.ElapsedMilliseconds / 1000), 2)); - await _scheduleJobSessionRepository.ManageAsync(updateResultCmd, ct); + await _scheduleJobSessionRepository.ManageAsync(updateResultCmd, cts.Token); } catch (Exception ex) { diff --git a/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs b/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs index 3ab2d9d6f8..d74845969f 100644 --- a/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs +++ b/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs @@ -36,7 +36,8 @@ public async Task RunJobSessionAsync( [FromRoute] Guid sessionId, CancellationToken ct) { - _mediator.Send(new RunScheduleJobSessionCommand(sessionId), ct); + int? concurrentRequests = _configuration.GetValue("ScheduleJobConcurrentRequests"); + _mediator.Send(new RunScheduleJobSessionCommand(sessionId, concurrentRequests ?? 5), ct); return Ok(); } } diff --git a/src/Spd.Presentation.Dynamics/appsettings.json b/src/Spd.Presentation.Dynamics/appsettings.json index 1647e012c2..ce8a7a78a2 100644 --- a/src/Spd.Presentation.Dynamics/appsettings.json +++ b/src/Spd.Presentation.Dynamics/appsettings.json @@ -12,6 +12,7 @@ "ScreeningAppPaymentPath": "api/crrpa/payment-secure-link", "LicensingAppPaymentPath": "api/licensing/payment-secure-link", "ScreeningOrgInvitationPath": "crrp/invitation-link-bceid", + "ScheduleJobConcurrentRequests": 3, //when the openshift call dyanmics actions, how many concurent requests can be sent out to dynamics. "PayBC": { "DirectRefund": { "AuthenticationSettings": { diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/Contract.cs b/src/Spd.Resource.Repository.JobSchedule/Org/Contract.cs index 0e2ccf499c..f89f0ba28c 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/Contract.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/Contract.cs @@ -4,6 +4,6 @@ namespace Spd.Resource.Repository.JobSchedule.Org { public interface IOrgRepository { - public Task> RunMonthlyInvoiceAsync(CancellationToken cancellationToken); + public Task> RunMonthlyInvoiceAsync(int concurrentRequests, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs index 4bc126b5e2..76117ab1ef 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs @@ -9,17 +9,70 @@ internal class OrgRepository : IOrgRepository private readonly DynamicsContext _context; private readonly IMapper _mapper; private readonly ILogger _logger; + private readonly IDynamicsContextFactory _dynamicsContextFactory; public OrgRepository(IDynamicsContextFactory ctx, IMapper mapper, ILogger logger) { + _dynamicsContextFactory = ctx; _context = ctx.Create(); _mapper = mapper; this._logger = logger; } - public async Task> RunMonthlyInvoiceAsync(CancellationToken ct) + //sequential + //public async Task> RunMonthlyInvoiceAsync(CancellationToken ct) + //{ + // int completed = 0; + // List results = new List(); + // var accounts = _context.accounts.Where(a => a.statecode == DynamicsConstants.StateCode_Active) + // .Where(a => a.spd_eligibleforcreditpayment == (int)YesNoOptionSet.Yes) + // .ToList(); + + // //delegate, for reporting progress + // void ReportProgress(int current) + // { + // if (current % 100 == 0 || current == accounts.Count()) + // { + // _logger.LogInformation($"Processed {current} of {accounts.Count()} accounts"); + // } + // } + // _logger.LogInformation("1 concurrent requests"); + + // foreach (var a in accounts) + // { + // try + // { + // var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); + // _logger.LogInformation($"Monthly Invoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); + // ResultResp rr = _mapper.Map(response); + // rr.PrimaryEntityId = a.accountid.Value; + // results.Add(rr); + // } + // catch (Exception ex) + // { + // _logger.LogError($"{a.accountid.Value}-{ex.Message}"); + // Exception current = ex; + // while (current != null) + // { + // Console.WriteLine($"Exception Type: {current.GetType().Name}"); + // Console.WriteLine($"Message: {current.Message}"); + // Console.WriteLine($"Stack Trace: {current.StackTrace}"); + // current = current.InnerException; + // } + // results.Add(new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }); + // } + // finally + // { + // int current = Interlocked.Increment(ref completed); + // ReportProgress(current); + // } + // } + // return results; + //} + + public async Task> RunMonthlyInvoiceAsync(int concurrentRequests, CancellationToken ct) { int completed = 0; var accounts = _context.accounts.Where(a => a.statecode == DynamicsConstants.StateCode_Active) @@ -35,15 +88,16 @@ void ReportProgress(int current) } } - using var semaphore = new SemaphoreSlim(2); // Limit to 2 concurrent requests - _logger.LogInformation("2 concurrent requests"); + using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to 1 concurrent requests + _logger.LogDebug($"{concurrentRequests} concurrent requests"); + var tasks = accounts.Select(async a => { await semaphore.WaitAsync(); try { var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); - _logger.LogInformation($"{response.IsSuccess} {response.Result} {a.accountid.Value}"); + _logger.LogInformation($"MonthlyInvoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); ResultResp rr = _mapper.Map(response); rr.PrimaryEntityId = a.accountid.Value; return rr; @@ -51,6 +105,14 @@ void ReportProgress(int current) catch (Exception ex) { _logger.LogError($"{a.accountid.Value}-{ex.Message}"); + Exception current = ex; + while (current != null) + { + Console.WriteLine($"Exception Type: {current.GetType().Name}"); + Console.WriteLine($"Message: {current.Message}"); + Console.WriteLine($"Stack Trace: {current.StackTrace}"); + current = current.InnerException; + } return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }; } finally @@ -64,7 +126,5 @@ void ReportProgress(int current) var results = await Task.WhenAll(tasks); return results; } - - } diff --git a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs index a9f70178cc..ccec0e26a1 100644 --- a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs +++ b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs @@ -18,7 +18,7 @@ public Mappings() _ = CreateMap() .ForMember(d => d.statecode, opt => opt.MapFrom(s => s.JobSessionStatusCode == JobSessionStatusCode.Success ? DynamicsConstants.StateCode_Inactive : DynamicsConstants.StateCode_Active)) .ForMember(d => d.statuscode, opt => opt.MapFrom(s => s.JobSessionStatusCode == JobSessionStatusCode.Success ? (int)BcGoV_ScheduleJObsession_StatusCode_OptionSet.Success : (int)BcGoV_ScheduleJObsession_StatusCode_OptionSet.Failed)) - .ForMember(d => d.bcgov_error, opt => opt.MapFrom(s => s.ErrorMsg)) + .ForMember(d => d.bcgov_error, opt => opt.MapFrom(s => s.ErrorMsg ?? string.Empty)) .ForMember(d => d.bcgov_duration, opt => opt.Ignore()); } } diff --git a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs index 66edb49364..55705af736 100644 --- a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs @@ -1,5 +1,6 @@ using AutoMapper; using Microsoft.Dynamics.CRM; +using Microsoft.Extensions.Logging; using Microsoft.OData.Client; using Spd.Utilities.Dynamics; @@ -8,11 +9,15 @@ internal class ScheduleJobSessionRepository : IScheduleJobSessionRepository { private readonly DynamicsContext _context; private readonly IMapper _mapper; + private readonly ILogger _logger; + public ScheduleJobSessionRepository(IDynamicsContextFactory ctx, - IMapper mapper) + IMapper mapper, + ILogger logger) { _context = ctx.Create(); _mapper = mapper; + _logger = logger; } public async Task GetAsync(Guid jobSessionId, CancellationToken ct) @@ -43,12 +48,13 @@ public async Task ManageAsync(UpdateScheduleJobSessionCm .FirstOrDefaultAsync(ct); if (jobSession == null) throw new ArgumentException("Invalid jobsession id, cannot find corresponding bcgov_schedulejobsession"); - _mapper.Map(updateCmd, jobSession); _context.UpdateObject(jobSession); await _context.SaveChangesAsync(ct); + _logger.LogInformation($"job session - {jobSession.bcgov_schedulejobsessionid} is updated."); return _mapper.Map(jobSession); } + public async Task QueryAsync(ScheduleJobSessionQry qry, CancellationToken ct) { throw new NotImplementedException(); diff --git a/src/Spd.Utilities.Dynamics/Configurer.cs b/src/Spd.Utilities.Dynamics/Configurer.cs index 37b940f55f..7d79c3a14e 100644 --- a/src/Spd.Utilities.Dynamics/Configurer.cs +++ b/src/Spd.Utilities.Dynamics/Configurer.cs @@ -18,8 +18,7 @@ public void Configure(ConfigurationContext configurationServices) services .AddHttpClient("oauth") - .SetHandlerLifetime(TimeSpan.FromMinutes(50)) - ; + .SetHandlerLifetime(TimeSpan.FromMinutes(2)); services.AddSingleton(); @@ -28,7 +27,7 @@ public void Configure(ConfigurationContext configurationServices) .AddODataClientHandler() .AddHttpClient() .ConfigureHttpClient(c => c.Timeout = options.HttpClientTimeout) - .SetHandlerLifetime(TimeSpan.FromMinutes(50)) + .SetHandlerLifetime(TimeSpan.FromMinutes(2)) ; services.AddSingleton(); From eca1cbd631a3d646ee5e74819f2cdf0064120581 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Thu, 1 May 2025 16:56:34 -0700 Subject: [PATCH 02/13] clean up --- src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs | 2 -- src/Spd.Utilities.Dynamics/Configurer.cs | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs index 76117ab1ef..96a0d941c1 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs @@ -9,13 +9,11 @@ internal class OrgRepository : IOrgRepository private readonly DynamicsContext _context; private readonly IMapper _mapper; private readonly ILogger _logger; - private readonly IDynamicsContextFactory _dynamicsContextFactory; public OrgRepository(IDynamicsContextFactory ctx, IMapper mapper, ILogger logger) { - _dynamicsContextFactory = ctx; _context = ctx.Create(); _mapper = mapper; this._logger = logger; diff --git a/src/Spd.Utilities.Dynamics/Configurer.cs b/src/Spd.Utilities.Dynamics/Configurer.cs index 7d79c3a14e..87f9ccca71 100644 --- a/src/Spd.Utilities.Dynamics/Configurer.cs +++ b/src/Spd.Utilities.Dynamics/Configurer.cs @@ -18,7 +18,7 @@ public void Configure(ConfigurationContext configurationServices) services .AddHttpClient("oauth") - .SetHandlerLifetime(TimeSpan.FromMinutes(2)); + .SetHandlerLifetime(TimeSpan.FromMinutes(50)); services.AddSingleton(); @@ -27,7 +27,7 @@ public void Configure(ConfigurationContext configurationServices) .AddODataClientHandler() .AddHttpClient() .ConfigureHttpClient(c => c.Timeout = options.HttpClientTimeout) - .SetHandlerLifetime(TimeSpan.FromMinutes(2)) + .SetHandlerLifetime(TimeSpan.FromMinutes(50)) ; services.AddSingleton(); From f6d4c5aee78172667af7dafb18ce2617f48cf972 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Fri, 2 May 2025 10:26:44 -0700 Subject: [PATCH 03/13] add paging --- .../Org/OrgRepository.cs | 53 ++++++++++++++----- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs index 96a0d941c1..8b787328cc 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs @@ -1,5 +1,7 @@ using AutoMapper; +using Microsoft.Dynamics.CRM; using Microsoft.Extensions.Logging; +using Microsoft.OData.Client; using Spd.Resource.Repository.JobSchedule.GeneralizeScheduleJob; using Spd.Utilities.Dynamics; @@ -24,9 +26,7 @@ public OrgRepository(IDynamicsContextFactory ctx, //{ // int completed = 0; // List results = new List(); - // var accounts = _context.accounts.Where(a => a.statecode == DynamicsConstants.StateCode_Active) - // .Where(a => a.spd_eligibleforcreditpayment == (int)YesNoOptionSet.Yes) - // .ToList(); + // var accounts = await GetAllAccountsAsync(ct); // //delegate, for reporting progress // void ReportProgress(int current) @@ -43,7 +43,7 @@ public OrgRepository(IDynamicsContextFactory ctx, // try // { // var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); - // _logger.LogInformation($"Monthly Invoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); + // _logger.LogDebug($"Monthly Invoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); // ResultResp rr = _mapper.Map(response); // rr.PrimaryEntityId = a.accountid.Value; // results.Add(rr); @@ -73,9 +73,7 @@ public OrgRepository(IDynamicsContextFactory ctx, public async Task> RunMonthlyInvoiceAsync(int concurrentRequests, CancellationToken ct) { int completed = 0; - var accounts = _context.accounts.Where(a => a.statecode == DynamicsConstants.StateCode_Active) - .Where(a => a.spd_eligibleforcreditpayment == (int)YesNoOptionSet.Yes) - .ToList(); + var accounts = await GetAllAccountsAsync(ct); //delegate, for reporting progress void ReportProgress(int current) @@ -86,7 +84,7 @@ void ReportProgress(int current) } } - using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to 1 concurrent requests + using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to n concurrent requests _logger.LogDebug($"{concurrentRequests} concurrent requests"); var tasks = accounts.Select(async a => @@ -95,7 +93,7 @@ void ReportProgress(int current) try { var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); - _logger.LogInformation($"MonthlyInvoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); + _logger.LogDebug($"MonthlyInvoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); ResultResp rr = _mapper.Map(response); rr.PrimaryEntityId = a.accountid.Value; return rr; @@ -106,9 +104,9 @@ void ReportProgress(int current) Exception current = ex; while (current != null) { - Console.WriteLine($"Exception Type: {current.GetType().Name}"); - Console.WriteLine($"Message: {current.Message}"); - Console.WriteLine($"Stack Trace: {current.StackTrace}"); + _logger.LogError($"Exception Type: {current.GetType().Name}"); + _logger.LogError($"Message: {current.Message}"); + _logger.LogError($"Stack Trace: {current.StackTrace}"); current = current.InnerException; } return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }; @@ -124,5 +122,36 @@ void ReportProgress(int current) var results = await Task.WhenAll(tasks); return results; } + + public async Task> GetAllAccountsAsync(CancellationToken ct) + { + int completed = 0; + string filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; + + var accountsQuery = _context.accounts + .AddQueryOption("$filter", filterStr) + .IncludeCount(); + + var allAccounts = new List(); + + QueryOperationResponse response; + DataServiceQueryContinuation continuation = null; + + do + { + if (continuation == null) + { + response = (QueryOperationResponse)await accountsQuery.ExecuteAsync(ct); + } + else + { + response = (QueryOperationResponse)await _context.ExecuteAsync(continuation, ct); + } + allAccounts.AddRange(response); + continuation = response.GetContinuation(); + } while (continuation != null); + + return allAccounts; + } } From fd4a056e60bb2fa54f2a5eceb0dad06df57bc274 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Fri, 2 May 2025 15:54:10 -0700 Subject: [PATCH 04/13] review --- .../Controllers/ScheduleJobController.cs | 6 +- .../Org/OrgRepository.cs | 63 ++----------------- 2 files changed, 10 insertions(+), 59 deletions(-) diff --git a/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs b/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs index d74845969f..929a95e410 100644 --- a/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs +++ b/src/Spd.Presentation.Dynamics/Controllers/ScheduleJobController.cs @@ -36,8 +36,10 @@ public async Task RunJobSessionAsync( [FromRoute] Guid sessionId, CancellationToken ct) { - int? concurrentRequests = _configuration.GetValue("ScheduleJobConcurrentRequests"); - _mediator.Send(new RunScheduleJobSessionCommand(sessionId, concurrentRequests ?? 5), ct); + int concurrentRequests = _configuration.GetValue("ScheduleJobConcurrentRequests"); +#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + _mediator.Send(new RunScheduleJobSessionCommand(sessionId, concurrentRequests), ct); +#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed return Ok(); } } diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs index 8b787328cc..95d8d531ef 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs @@ -20,56 +20,6 @@ public OrgRepository(IDynamicsContextFactory ctx, _mapper = mapper; this._logger = logger; } - - //sequential - //public async Task> RunMonthlyInvoiceAsync(CancellationToken ct) - //{ - // int completed = 0; - // List results = new List(); - // var accounts = await GetAllAccountsAsync(ct); - - // //delegate, for reporting progress - // void ReportProgress(int current) - // { - // if (current % 100 == 0 || current == accounts.Count()) - // { - // _logger.LogInformation($"Processed {current} of {accounts.Count()} accounts"); - // } - // } - // _logger.LogInformation("1 concurrent requests"); - - // foreach (var a in accounts) - // { - // try - // { - // var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); - // _logger.LogDebug($"Monthly Invoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); - // ResultResp rr = _mapper.Map(response); - // rr.PrimaryEntityId = a.accountid.Value; - // results.Add(rr); - // } - // catch (Exception ex) - // { - // _logger.LogError($"{a.accountid.Value}-{ex.Message}"); - // Exception current = ex; - // while (current != null) - // { - // Console.WriteLine($"Exception Type: {current.GetType().Name}"); - // Console.WriteLine($"Message: {current.Message}"); - // Console.WriteLine($"Stack Trace: {current.StackTrace}"); - // current = current.InnerException; - // } - // results.Add(new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }); - // } - // finally - // { - // int current = Interlocked.Increment(ref completed); - // ReportProgress(current); - // } - // } - // return results; - //} - public async Task> RunMonthlyInvoiceAsync(int concurrentRequests, CancellationToken ct) { int completed = 0; @@ -80,12 +30,12 @@ void ReportProgress(int current) { if (current % 100 == 0 || current == accounts.Count()) { - _logger.LogInformation($"Processed {current} of {accounts.Count()} accounts"); + _logger.LogInformation("Processed {Current} of {AccountsCount()} accounts", current, accounts.Count()); } } using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to n concurrent requests - _logger.LogDebug($"{concurrentRequests} concurrent requests"); + _logger.LogDebug("{ConcurrentRequests} concurrent requests", concurrentRequests); var tasks = accounts.Select(async a => { @@ -93,7 +43,7 @@ void ReportProgress(int current) try { var response = await a.spd_MonthlyInvoice().GetValueAsync(ct); - _logger.LogDebug($"MonthlyInvoice executed result : success = {response.IsSuccess} {response.Result} accountid={a.accountid.Value}"); + _logger.LogDebug("MonthlyInvoice executed result : success = {Success} {Result} accountid={Accountid}", response.IsSuccess, response.Result, a.accountid.Value); ResultResp rr = _mapper.Map(response); rr.PrimaryEntityId = a.accountid.Value; return rr; @@ -104,9 +54,9 @@ void ReportProgress(int current) Exception current = ex; while (current != null) { - _logger.LogError($"Exception Type: {current.GetType().Name}"); - _logger.LogError($"Message: {current.Message}"); - _logger.LogError($"Stack Trace: {current.StackTrace}"); + _logger.LogError("Exception Type: {ExceptionName}", current.GetType().Name); + _logger.LogError("Message: {Message}", current.Message); + _logger.LogError("Stack Trace: {StackTrace}", current.StackTrace); current = current.InnerException; } return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }; @@ -125,7 +75,6 @@ void ReportProgress(int current) public async Task> GetAllAccountsAsync(CancellationToken ct) { - int completed = 0; string filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; var accountsQuery = _context.accounts From aefb62d2813ca1c19983132f9381c65a0ef264f4 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Fri, 2 May 2025 15:56:44 -0700 Subject: [PATCH 05/13] log fix --- .../ScheduleJobSession/ScheduleJobSessionRepository.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs index 55705af736..fdb0ef233b 100644 --- a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/ScheduleJobSessionRepository.cs @@ -51,7 +51,7 @@ public async Task ManageAsync(UpdateScheduleJobSessionCm _mapper.Map(updateCmd, jobSession); _context.UpdateObject(jobSession); await _context.SaveChangesAsync(ct); - _logger.LogInformation($"job session - {jobSession.bcgov_schedulejobsessionid} is updated."); + _logger.LogInformation("job session - {JobSessionId} is updated.", jobSession.bcgov_schedulejobsessionid); return _mapper.Map(jobSession); } From ad52489477b41a7b1a422a5042ba6b83fcfbba17 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Fri, 2 May 2025 16:08:02 -0700 Subject: [PATCH 06/13] sonar --- .../Org/OrgRepository.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs index 95d8d531ef..3fd2d91bc3 100644 --- a/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/Org/OrgRepository.cs @@ -30,7 +30,7 @@ void ReportProgress(int current) { if (current % 100 == 0 || current == accounts.Count()) { - _logger.LogInformation("Processed {Current} of {AccountsCount()} accounts", current, accounts.Count()); + _logger.LogInformation("Processed {Current} of {AccountsCount} accounts", current, accounts.Count()); } } @@ -50,13 +50,10 @@ void ReportProgress(int current) } catch (Exception ex) { - _logger.LogError($"{a.accountid.Value}-{ex.Message}"); Exception current = ex; while (current != null) { - _logger.LogError("Exception Type: {ExceptionName}", current.GetType().Name); - _logger.LogError("Message: {Message}", current.Message); - _logger.LogError("Stack Trace: {StackTrace}", current.StackTrace); + _logger.LogError("Exception Type: {ExceptionName} \r\n Message: {Message} \r\n Stack Trace: {StackTrace}", current.GetType().Name, current.Message, current.StackTrace); current = current.InnerException; } return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = a.accountid.Value }; From 699edaffb557fcbcbe966f35f1e27396dbbbefbd Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Fri, 2 May 2025 17:52:05 -0700 Subject: [PATCH 07/13] general --- .../GeneralizeScheduleJobRepository.cs | 160 +++++++++++++++++- 1 file changed, 152 insertions(+), 8 deletions(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index 10d11f20d9..9cbeb57de1 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -1,6 +1,7 @@ -using AutoMapper; +using AutoMapper; using Microsoft.OData.Client; using Spd.Utilities.Dynamics; +using System.Reflection; namespace Spd.Resource.Repository.JobSchedule.GeneralizeScheduleJob; internal class GeneralizeScheduleJobRepository : IGeneralizeScheduleJobRepository @@ -14,6 +15,66 @@ public GeneralizeScheduleJobRepository(IDynamicsContextFactory ctx, _mapper = mapper; } + //public async Task> RunJobsAsync(RunJobRequest request, CancellationToken ct) + //{ + // string primaryEntityName = request.PrimaryEntityName; + // string filterStr = request.PrimaryEntityFilterStr; + // string actionStr = request.PrimaryEntityActionStr; + // string primaryEntityIdName = request.PrimaryEntityIdName; + + // var property = _context.GetType().GetProperty(primaryEntityName); + // if (property == null) throw new Exception("Property not found."); + + // dynamic query = property.GetValue(_context) as DataServiceQuery; + // query.AddQueryOption("$filter", $"{filterStr}"); + // var result = await query.ExecuteAsync(ct); + // var data = ((IEnumerable)result).ToList(); + + // using var semaphore = new SemaphoreSlim(10); // Limit to 10 concurrent requests + + // var tasks = data.Select(async a => + // { + // await semaphore.WaitAsync(); + // try + // { + // //get primary id + // var idProperty = a.GetType().GetProperty(primaryEntityIdName); + // object obj = idProperty.GetValue(a); + // string idStr = obj.ToString(); + + // //invoke method + // var method = a.GetType().GetMethod(actionStr); + // var result = method?.Invoke(a, null); + // var getValueAsyncMethod = result.GetType().GetMethod( + // "GetValueAsync", + // new[] { typeof(CancellationToken) }); + // if (getValueAsyncMethod == null) throw new Exception("GetValueAsync method not found."); + // var task = (Task)getValueAsyncMethod.Invoke(result, new object[] { ct }); + // await task.ConfigureAwait(false); + + // // If it's Task, get the result via reflection + // var resultProperty = task.GetType().GetProperty("Result"); + // var value = resultProperty?.GetValue(task); + // ResultResp rr = _mapper.Map(value); + // rr.PrimaryEntityId = Guid.Parse(idStr); + // return rr; + // } + // catch (Exception ex) + // { + // var idProperty = a.GetType().GetProperty(primaryEntityIdName); + // object obj = idProperty.GetValue(a); + // return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = Guid.Parse(obj.ToString()) }; + // } + // finally + // { + // semaphore.Release(); + // } + // }); + + // var results = await Task.WhenAll(tasks); + // return results; + //} + public async Task> RunJobsAsync(RunJobRequest request, CancellationToken ct) { string primaryEntityName = request.PrimaryEntityName; @@ -21,13 +82,10 @@ public async Task> RunJobsAsync(RunJobRequest request, C string actionStr = request.PrimaryEntityActionStr; string primaryEntityIdName = request.PrimaryEntityIdName; - var property = _context.GetType().GetProperty(primaryEntityName); - if (property == null) throw new Exception("Property not found."); - - dynamic query = property.GetValue(_context) as DataServiceQuery; - query.AddQueryOption("$filter", $"{filterStr}"); - var result = await query.ExecuteAsync(ct); - var data = ((IEnumerable)result).ToList(); + string primaryTypeName = "account"; + // Resolve type dynamically + Type entityType = GetEntityTypeByName(primaryTypeName); + var data = await CallGetAllPrimaryEntityDynamicAsync(entityType, primaryEntityName, filterStr, ct); using var semaphore = new SemaphoreSlim(10); // Limit to 10 concurrent requests @@ -74,5 +132,91 @@ public async Task> RunJobsAsync(RunJobRequest request, C return results; } + private static Type GetEntityTypeByName(string typeName) + { + var type = Type.GetType(typeName); + if (type != null) + return type; + + type = AppDomain.CurrentDomain + .GetAssemblies() + .SelectMany(a => a.GetTypes()) + .FirstOrDefault(t => t.Name.Equals(typeName, StringComparison.OrdinalIgnoreCase)); + + if (type != null) + return type; + + throw new InvalidOperationException($"Type '{typeName}' not found."); + } + + private async Task> CallGetAllPrimaryEntityDynamicAsync( + Type entityType, + string primaryEntityName, + string filterStr, + CancellationToken ct) + { + var method = this.GetType().GetMethod( + "GetAllPrimaryEntityAsync", + BindingFlags.NonPublic | BindingFlags.Instance); + + if (method == null) + throw new InvalidOperationException("GetAllPrimaryEntityAsync method not found."); + + var genericMethod = method.MakeGenericMethod(entityType); + + var task = (Task)genericMethod.Invoke(this, new object[] { primaryEntityName, filterStr, ct }); + + await task.ConfigureAwait(false); + + var resultProperty = task.GetType().GetProperty("Result"); + var result = resultProperty.GetValue(task); + + // Convert IEnumerable → IEnumerable + var enumerable = result as System.Collections.IEnumerable; + if (enumerable == null) + throw new InvalidOperationException("The result is not enumerable."); + + var list = new List(); + foreach (var item in enumerable) + { + list.Add(item); + } + + return list; + } + + + + private async Task> GetAllPrimaryEntityAsync(string primaryEntityName, string filterStr, CancellationToken ct) + { + filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; + + var property = _context.GetType().GetProperty(primaryEntityName); + if (property == null) throw new Exception("Property not found."); + + dynamic query = property.GetValue(_context) as DataServiceQuery; + query.AddQueryOption("$filter", $"{filterStr}") + .IncludeCount(); + + var allEntities = new List(); + QueryOperationResponse response; + DataServiceQueryContinuation continuation = null; + + do + { + if (continuation == null) + { + response = (QueryOperationResponse)await query.ExecuteAsync(ct); + } + else + { + response = (QueryOperationResponse)await _context.ExecuteAsync(continuation, ct); + } + allEntities.AddRange(response); + continuation = response.GetContinuation(); + } while (continuation != null); + + return allEntities; + } } From aa19de2d7e6f7b41b5b3533c2aadf244f796340c Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Mon, 5 May 2025 09:00:14 -0700 Subject: [PATCH 08/13] change --- src/Spd.Manager.ScheduleJob/Contract.cs | 2 + .../ScheduleJobManager.cs | 51 ++++++++++--------- .../GeneralizeScheduleJobRepository.cs | 4 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/Spd.Manager.ScheduleJob/Contract.cs b/src/Spd.Manager.ScheduleJob/Contract.cs index e8d6be304f..d6a369095d 100644 --- a/src/Spd.Manager.ScheduleJob/Contract.cs +++ b/src/Spd.Manager.ScheduleJob/Contract.cs @@ -5,9 +5,11 @@ namespace Spd.Manager.ScheduleJob public interface IScheduleJobManager { public Task Handle(RunScheduleJobSessionCommand command, CancellationToken ct); + public Task Handle(RunMonthlyInvoiceJobCommand command, CancellationToken ct); } #region run schedule job session public record RunScheduleJobSessionCommand(Guid JobSessionId, int ConcurrentRequests) : IRequest; + public record RunMonthlyInvoiceJobCommand(Guid JobSessionId, int ConcurrentRequests) : IRequest; #endregion } diff --git a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs index e1e72ac38e..33c659b520 100644 --- a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs +++ b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs @@ -13,6 +13,7 @@ namespace Spd.Manager.ScheduleJob; public class ScheduleJobManager : IRequestHandler, + IRequestHandler, IScheduleJobManager { private readonly IScheduleJobSessionRepository _scheduleJobSessionRepository; @@ -31,7 +32,7 @@ public ScheduleJobManager(IScheduleJobSessionRepository scheduleJobSessionReposi _logger = logger; } - public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationToken ct) + public async Task Handle(RunMonthlyInvoiceJobCommand cmd, CancellationToken ct) { ScheduleJobSessionResp? resp = await _scheduleJobSessionRepository.GetAsync(cmd.JobSessionId, ct); if (resp == null) @@ -67,33 +68,33 @@ public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationTok return default; } - //generalized,waiting for mano testing - //public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationToken ct) - //{ - // ScheduleJobSessionResp? resp = await _scheduleJobSessionRepository.GetAsync(cmd.JobSessionId, ct); - // if (resp == null) - // { - // throw new ApiException(HttpStatusCode.BadRequest, "The schedule job session does not exist."); - // } + public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationToken ct) + { + ScheduleJobSessionResp? resp = await _scheduleJobSessionRepository.GetAsync(cmd.JobSessionId, ct); + if (resp == null) + { + throw new ApiException(HttpStatusCode.BadRequest, "The schedule job session does not exist."); + } - // Stopwatch stopwatch = Stopwatch.StartNew(); - // //request will from bcgov_schedulejob - // RunJobRequest request = new RunJobRequest - // { - // PrimaryEntityActionStr = "spd_MonthlyInvoice", - // PrimaryEntityName = "accounts", - // PrimaryEntityFilterStr = "statecode eq 0 && spd_eligibleforcreditpayment eq 100000001", - // PrimaryEntityIdName = "accountid" - // }; - // var result = await _generalizeScheduleJobRepository.RunJobsAsync(request, ct); - // stopwatch.Stop(); + Stopwatch stopwatch = Stopwatch.StartNew(); + using var cts = new CancellationTokenSource(); // no timeout + //request will from bcgov_schedulejob + RunJobRequest request = new RunJobRequest + { + PrimaryEntityActionStr = "spd_MonthlyInvoice", + PrimaryEntityName = "accounts", + PrimaryEntityFilterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001", + PrimaryEntityIdName = "accountid" + }; + var result = await _generalizeScheduleJobRepository.RunJobsAsync(request, cts.Token); + stopwatch.Stop(); - // //update result in JobSession - // UpdateScheduleJobSessionCmd updateResultCmd = CreateUpdateScheduleJobSessionCmd(cmd.JobSessionId, result, Decimal.Round((decimal)(stopwatch.ElapsedMilliseconds / 1000), 2)); - // await _scheduleJobSessionRepository.ManageAsync(updateResultCmd, ct); + //update result in JobSession + UpdateScheduleJobSessionCmd updateResultCmd = CreateUpdateScheduleJobSessionCmd(cmd.JobSessionId, result, Decimal.Round((decimal)(stopwatch.ElapsedMilliseconds / 1000), 2)); + await _scheduleJobSessionRepository.ManageAsync(updateResultCmd, cts.Token); - // return default; - //} + return default; + } private UpdateScheduleJobSessionCmd CreateUpdateScheduleJobSessionCmd(Guid jobSessionId, IEnumerable results, decimal durationInSec) { diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index 9cbeb57de1..132c982672 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -185,8 +185,6 @@ private async Task> CallGetAllPrimaryEntityDynamicAsync( return list; } - - private async Task> GetAllPrimaryEntityAsync(string primaryEntityName, string filterStr, CancellationToken ct) { filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; @@ -195,7 +193,7 @@ private async Task> GetAllPrimaryEntityAsync(string primaryEnt if (property == null) throw new Exception("Property not found."); dynamic query = property.GetValue(_context) as DataServiceQuery; - query.AddQueryOption("$filter", $"{filterStr}") + query.AddQueryOption("$filter", filterStr) .IncludeCount(); var allEntities = new List(); From 9152fa4966808995830e6495222b1919ae3c6c81 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Mon, 5 May 2025 11:24:59 -0700 Subject: [PATCH 09/13] change --- src/Spd.Manager.ScheduleJob/Contract.cs | 4 ++-- src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs | 4 ++-- .../GeneralizeScheduleJob/Contract.cs | 2 +- .../GeneralizeScheduleJobRepository.cs | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Spd.Manager.ScheduleJob/Contract.cs b/src/Spd.Manager.ScheduleJob/Contract.cs index d6a369095d..41f0d98efd 100644 --- a/src/Spd.Manager.ScheduleJob/Contract.cs +++ b/src/Spd.Manager.ScheduleJob/Contract.cs @@ -9,7 +9,7 @@ public interface IScheduleJobManager } #region run schedule job session - public record RunScheduleJobSessionCommand(Guid JobSessionId, int ConcurrentRequests) : IRequest; - public record RunMonthlyInvoiceJobCommand(Guid JobSessionId, int ConcurrentRequests) : IRequest; + public record RunScheduleJobSessionCommand(Guid JobSessionId, int ConcurrentRequests = 3) : IRequest; + public record RunMonthlyInvoiceJobCommand(Guid JobSessionId, int ConcurrentRequests = 3) : IRequest; #endregion } diff --git a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs index 33c659b520..b0f7b0d75a 100644 --- a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs +++ b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs @@ -77,7 +77,7 @@ public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationTok } Stopwatch stopwatch = Stopwatch.StartNew(); - using var cts = new CancellationTokenSource(); // no timeout + using var cts = new CancellationTokenSource(); // no timeout, this is the resolve for mysteriously cancelled requests //request will from bcgov_schedulejob RunJobRequest request = new RunJobRequest { @@ -86,7 +86,7 @@ public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationTok PrimaryEntityFilterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001", PrimaryEntityIdName = "accountid" }; - var result = await _generalizeScheduleJobRepository.RunJobsAsync(request, cts.Token); + var result = await _generalizeScheduleJobRepository.RunJobsAsync(request, cmd.ConcurrentRequests, cts.Token); stopwatch.Stop(); //update result in JobSession diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs index de0809e407..61c0a9fc87 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs @@ -2,7 +2,7 @@ namespace Spd.Resource.Repository.JobSchedule.GeneralizeScheduleJob { public interface IGeneralizeScheduleJobRepository { - public Task> RunJobsAsync(RunJobRequest request, CancellationToken cancellationToken); + public Task> RunJobsAsync(RunJobRequest request, int ConcurrentRequests, CancellationToken cancellationToken); } public record RunJobRequest diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index 132c982672..2d28a0903b 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -75,7 +75,7 @@ public GeneralizeScheduleJobRepository(IDynamicsContextFactory ctx, // return results; //} - public async Task> RunJobsAsync(RunJobRequest request, CancellationToken ct) + public async Task> RunJobsAsync(RunJobRequest request, int concurrentRequests, CancellationToken ct) { string primaryEntityName = request.PrimaryEntityName; string filterStr = request.PrimaryEntityFilterStr; @@ -87,7 +87,7 @@ public async Task> RunJobsAsync(RunJobRequest request, C Type entityType = GetEntityTypeByName(primaryTypeName); var data = await CallGetAllPrimaryEntityDynamicAsync(entityType, primaryEntityName, filterStr, ct); - using var semaphore = new SemaphoreSlim(10); // Limit to 10 concurrent requests + using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to 10 concurrent requests var tasks = data.Select(async a => { @@ -192,8 +192,8 @@ private async Task> GetAllPrimaryEntityAsync(string primaryEnt var property = _context.GetType().GetProperty(primaryEntityName); if (property == null) throw new Exception("Property not found."); - dynamic query = property.GetValue(_context) as DataServiceQuery; - query.AddQueryOption("$filter", filterStr) + var query = property.GetValue(_context) as DataServiceQuery; + query = query.AddQueryOption("$filter", filterStr) .IncludeCount(); var allEntities = new List(); From 01fe009feda16d3a053d2a080d8fe7cbc9961b51 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Mon, 5 May 2025 12:27:35 -0700 Subject: [PATCH 10/13] test --- src/Spd.Presentation.Dynamics/appsettings.json | 2 +- .../GeneralizeScheduleJobRepository.cs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Spd.Presentation.Dynamics/appsettings.json b/src/Spd.Presentation.Dynamics/appsettings.json index ce8a7a78a2..8b245f5cd2 100644 --- a/src/Spd.Presentation.Dynamics/appsettings.json +++ b/src/Spd.Presentation.Dynamics/appsettings.json @@ -12,7 +12,7 @@ "ScreeningAppPaymentPath": "api/crrpa/payment-secure-link", "LicensingAppPaymentPath": "api/licensing/payment-secure-link", "ScreeningOrgInvitationPath": "crrp/invitation-link-bceid", - "ScheduleJobConcurrentRequests": 3, //when the openshift call dyanmics actions, how many concurent requests can be sent out to dynamics. + "ScheduleJobConcurrentRequests": 10, //when the openshift call dyanmics actions, how many concurent requests can be sent out to dynamics. "PayBC": { "DirectRefund": { "AuthenticationSettings": { diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index 2d28a0903b..a8e238d2b9 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -1,4 +1,5 @@ using AutoMapper; +using Microsoft.Extensions.Logging; using Microsoft.OData.Client; using Spd.Utilities.Dynamics; using System.Reflection; @@ -8,11 +9,15 @@ internal class GeneralizeScheduleJobRepository : IGeneralizeScheduleJobRepositor { private readonly DynamicsContext _context; private readonly IMapper _mapper; + private readonly ILogger _logger; + public GeneralizeScheduleJobRepository(IDynamicsContextFactory ctx, - IMapper mapper) + IMapper mapper, + ILogger logger) { _context = ctx.Create(); _mapper = mapper; + this._logger = logger; } //public async Task> RunJobsAsync(RunJobRequest request, CancellationToken ct) @@ -88,6 +93,7 @@ public async Task> RunJobsAsync(RunJobRequest request, i var data = await CallGetAllPrimaryEntityDynamicAsync(entityType, primaryEntityName, filterStr, ct); using var semaphore = new SemaphoreSlim(concurrentRequests); // Limit to 10 concurrent requests + _logger.LogDebug("{ConcurrentRequests} concurrent requests", concurrentRequests); var tasks = data.Select(async a => { From cd59f7a699c59b6d7f3476dcca6f42fbbb42bc6c Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Mon, 5 May 2025 13:04:50 -0700 Subject: [PATCH 11/13] add log --- .../GeneralizeScheduleJobRepository.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index a8e238d2b9..c28ef3b073 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -120,10 +120,17 @@ public async Task> RunJobsAsync(RunJobRequest request, i var value = resultProperty?.GetValue(task); ResultResp rr = _mapper.Map(value); rr.PrimaryEntityId = Guid.Parse(idStr); + _logger.LogInformation("{actionStr} executed result : success = {Success} {Result} primaryEntityId={entityId}", actionStr, rr.IsSuccess, rr.ResultStr, idStr); return rr; } catch (Exception ex) { + Exception current = ex; + while (current != null) + { + _logger.LogError("Exception Type: {ExceptionName} \r\n Message: {Message} \r\n Stack Trace: {StackTrace}", current.GetType().Name, current.Message, current.StackTrace); + current = current.InnerException; + } var idProperty = a.GetType().GetProperty(primaryEntityIdName); object obj = idProperty.GetValue(a); return new ResultResp { IsSuccess = false, ResultStr = ex.Message, PrimaryEntityId = Guid.Parse(obj.ToString()) }; @@ -187,7 +194,7 @@ private async Task> CallGetAllPrimaryEntityDynamicAsync( { list.Add(item); } - + _logger.LogInformation("{Count} {Name} found", list.Count, primaryEntityName); return list; } From 5c349e1f847b0bb5985c5bfffd98e89f4acffb14 Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Mon, 5 May 2025 17:12:22 -0700 Subject: [PATCH 12/13] complete --- src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs | 10 ++++++---- .../GeneralizeScheduleJob/Contract.cs | 5 +++-- .../GeneralizeScheduleJobRepository.cs | 7 +++---- .../ScheduleJobSession/Contract.cs | 2 +- .../ScheduleJobSession/Mappings.cs | 1 + 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs index b0f7b0d75a..7c76dcb1a6 100644 --- a/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs +++ b/src/Spd.Manager.ScheduleJob/ScheduleJobManager.cs @@ -79,12 +79,14 @@ public async Task Handle(RunScheduleJobSessionCommand cmd, CancellationTok Stopwatch stopwatch = Stopwatch.StartNew(); using var cts = new CancellationTokenSource(); // no timeout, this is the resolve for mysteriously cancelled requests //request will from bcgov_schedulejob + //filterStr should be like "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001" RunJobRequest request = new RunJobRequest { - PrimaryEntityActionStr = "spd_MonthlyInvoice", - PrimaryEntityName = "accounts", - PrimaryEntityFilterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001", - PrimaryEntityIdName = "accountid" + PrimaryTypeName = resp.PrimaryEntity, + PrimaryEntityActionStr = resp.EndPoint, + PrimaryEntityName = resp.PrimaryEntity + "s", + PrimaryEntityFilterStr = resp.FilterStr.Trim(), + PrimaryEntityIdName = resp.PrimaryEntity + "id", }; var result = await _generalizeScheduleJobRepository.RunJobsAsync(request, cmd.ConcurrentRequests, cts.Token); stopwatch.Stop(); diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs index 61c0a9fc87..ebf236e6d0 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/Contract.cs @@ -7,8 +7,9 @@ public interface IGeneralizeScheduleJobRepository public record RunJobRequest { - public String PrimaryEntityName { get; set; } - public string PrimaryEntityIdName { get; set; } + public string PrimaryTypeName { get; set; } //exp: account + public String PrimaryEntityName { get; set; } //exp: accounts + public string PrimaryEntityIdName { get; set; } //exp: accountid public string? PrimaryEntityFilterStr { get; set; } public string PrimaryEntityActionStr { get; set; } } diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index c28ef3b073..7342202104 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -82,12 +82,12 @@ public GeneralizeScheduleJobRepository(IDynamicsContextFactory ctx, public async Task> RunJobsAsync(RunJobRequest request, int concurrentRequests, CancellationToken ct) { + string primaryTypeName = request.PrimaryTypeName; string primaryEntityName = request.PrimaryEntityName; - string filterStr = request.PrimaryEntityFilterStr; + string filterStr = request.PrimaryEntityFilterStr ?? string.Empty; string actionStr = request.PrimaryEntityActionStr; string primaryEntityIdName = request.PrimaryEntityIdName; - string primaryTypeName = "account"; // Resolve type dynamically Type entityType = GetEntityTypeByName(primaryTypeName); var data = await CallGetAllPrimaryEntityDynamicAsync(entityType, primaryEntityName, filterStr, ct); @@ -200,8 +200,7 @@ private async Task> CallGetAllPrimaryEntityDynamicAsync( private async Task> GetAllPrimaryEntityAsync(string primaryEntityName, string filterStr, CancellationToken ct) { - filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; - + //filterStr = "statecode eq 0 and spd_eligibleforcreditpayment eq 100000001"; var property = _context.GetType().GetProperty(primaryEntityName); if (property == null) throw new Exception("Property not found."); diff --git a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Contract.cs b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Contract.cs index bc3897b656..9a8f65b08a 100644 --- a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Contract.cs +++ b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Contract.cs @@ -29,7 +29,7 @@ public record ScheduleJobSessionResp() public Guid ScheduleJobId { get; set; } public string EndPoint { get; set; } public string PrimaryEntity { get; set; } - + public string FilterStr { get; set; } } public enum JobSessionStatusCode diff --git a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs index ccec0e26a1..94d60cc134 100644 --- a/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs +++ b/src/Spd.Resource.Repository.JobSchedule/ScheduleJobSession/Mappings.cs @@ -13,6 +13,7 @@ public Mappings() .ForMember(d => d.ScheduleJobId, opt => opt.MapFrom(s => s._bcgov_schedulejobid_value)) .ForMember(d => d.PrimaryEntity, opt => opt.MapFrom(s => s.bcgov_ScheduleJobId.bcgov_primaryentity)) .ForMember(d => d.EndPoint, opt => opt.MapFrom(s => s.bcgov_ScheduleJobId.bcgov_endpoint)) + .ForMember(d => d.FilterStr, opt => opt.MapFrom(s => s.bcgov_ScheduleJobId.bcgov_fetchxml)) ; _ = CreateMap() From f8926368ca9136d6259e1f2b4cd06abca193343a Mon Sep 17 00:00:00 2001 From: peggy-quartech Date: Tue, 6 May 2025 10:05:17 -0700 Subject: [PATCH 13/13] respond to sonar --- .../GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs index 7342202104..b8379a3dc0 100644 --- a/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs +++ b/src/Spd.Resource.Repository.JobSchedule/GeneralizeScheduleJob/GeneralizeScheduleJobRepository.cs @@ -1,4 +1,4 @@ -using AutoMapper; +using AutoMapper; using Microsoft.Extensions.Logging; using Microsoft.OData.Client; using Spd.Utilities.Dynamics; @@ -108,10 +108,13 @@ public async Task> RunJobsAsync(RunJobRequest request, i //invoke method var method = a.GetType().GetMethod(actionStr); var result = method?.Invoke(a, null); + if (result == null) throw new Exception($"the {actionStr} invoked returns null"); + var getValueAsyncMethod = result.GetType().GetMethod( "GetValueAsync", new[] { typeof(CancellationToken) }); if (getValueAsyncMethod == null) throw new Exception("GetValueAsync method not found."); + var task = (Task)getValueAsyncMethod.Invoke(result, new object[] { ct }); await task.ConfigureAwait(false);