A customer received very large error reports (100Mb JSON) for certain errors. Those are now discarded.

Changed so that the new ReportReceivedAt property is used when displaying incidents.

Author: jgauffin

src/Server/Coderr.Server.Api/Core/Incidents/Queries/FindIncidentsResultItem.cs

@@ -54,7 +54,7 @@ protected FindIncidentsResultItem()
         public bool IsReOpened { get; set; }
 
         /// <summary>
-        ///     When the last report was received (or when the last user action was made)
+        ///     When someone updated this incident (assigned/closed etc).
         /// </summary>
         public DateTime LastUpdateAtUtc { get; set; }
 
@@ -67,5 +67,10 @@ protected FindIncidentsResultItem()
         ///     Total number of received reports (increased even if the number of stored reports are at the limit)
         /// </summary>
         public int ReportCount { get; set; }
+
+        /// <summary>
+        /// When we recieved the last report.
+        /// </summary>
+        public DateTime LastReportReceivedAtUtc { get; set; }
     }
 }

src/Server/Coderr.Server.Api/Core/Incidents/Queries/GetIncidentResult.cs

@@ -127,6 +127,10 @@ public string Description
         /// </summary>
         public int ReportCount { get; set; }
 
+        /// <summary>
+        /// When we received the last report for this incident.
+        /// </summary>
+        public DateTime LastReportReceivedAtUtc { get; set; }
 
         /// <summary>
         ///     Generated hash code

src/Server/Coderr.Server.App/Core/Reports/Config/ReportConfig.cs

@@ -8,6 +8,16 @@ namespace codeRR.Server.App.Core.Reports.Config
     /// </summary>
     public class ReportConfig : IConfigurationSection
     {
+        /// <summary>
+        ///     Maximum number of bytes that a uncompressed JSON report can be
+        /// </summary>
+        /// <remarks>
+        ///     <para>
+        ///         Used to filter out reports that are too large.
+        ///     </para>
+        /// </remarks>
+        public int MaxReportJsonSize { get; set; }
+
         /// <summary>
         ///     Max number of reports per incident
         /// </summary>
@@ -24,11 +34,7 @@ public class ReportConfig : IConfigurationSection
         /// </remarks>
         public int RetentionDays { get; set; }
 
-
-        string IConfigurationSection.SectionName
-        {
-            get { return "ReportConfig"; }
-        }
+        string IConfigurationSection.SectionName => "ReportConfig";
 
         IDictionary<string, string> IConfigurationSection.ToDictionary()
         {
@@ -38,6 +44,8 @@ IDictionary<string, string> IConfigurationSection.ToDictionary()
         void IConfigurationSection.Load(IDictionary<string, string> settings)
         {
             this.AssignProperties(settings);
+            if (MaxReportJsonSize == 0)
+                MaxReportJsonSize = 1000000;
         }
     }
 }

src/Server/Coderr.Server.Infrastructure/Configuration/ConfigurationStore.cs

@@ -1,6 +1,4 @@
 using System;
-using System.Diagnostics.CodeAnalysis;
-using codeRR.Server.Infrastructure.Configuration.Database;
 
 namespace codeRR.Server.Infrastructure.Configuration
 {

src/Server/Coderr.Server.ReportAnalyzer/Handlers/Reports/ReportAnalyzer.cs

@@ -62,7 +62,10 @@ public void Analyze(ClaimsPrincipal user, ErrorReportEntity report)
             }
             catch (Exception ex)
             {
-                _logger.Fatal("Failed to store report " + JsonConvert.SerializeObject(report), ex);
+                var reportJson = JsonConvert.SerializeObject(report);
+                if (reportJson.Length > 1000000)
+                    reportJson = reportJson.Substring(0, 100000) + "[....]";
+                _logger.Fatal("Failed to init report " + reportJson, ex);
                 return;
             }
 

src/Server/Coderr.Server.ReportAnalyzer/Inbound/SaveReportHandler.cs

@@ -7,13 +7,13 @@
 using System.Security.Claims;
 using System.Text;
 using System.Threading.Tasks;
-using codeRR.Server.Infrastructure;
+using codeRR.Server.App.Core.Reports.Config;
+using codeRR.Server.Infrastructure.Configuration;
 using codeRR.Server.ReportAnalyzer.Inbound.Models;
 using codeRR.Server.ReportAnalyzer.LibContracts;
 using DotNetCqs;
 using DotNetCqs.Queues;
 using Griffin.Data;
-using Griffin.Data.Mapper;
 using log4net;
 using Newtonsoft.Json;
 
@@ -24,20 +24,23 @@ namespace codeRR.Server.ReportAnalyzer.Inbound
     /// </summary>
     public class SaveReportHandler
     {
-        private readonly IAdoNetUnitOfWork _unitOfWork;
+        private readonly List<Func<NewReportDTO, bool>> _filters = new List<Func<NewReportDTO, bool>>();
         private readonly ILog _logger = LogManager.GetLogger(typeof(SaveReportHandler));
         private readonly IMessageQueue _queue;
-        private readonly List<Func<NewReportDTO, bool>> _filters = new List<Func<NewReportDTO, bool>>();
+        private readonly IAdoNetUnitOfWork _unitOfWork;
+        private readonly int _maxSizeForJsonErrorReport;
 
         /// <summary>
         ///     Creates a new instance of <see cref="SaveReportHandler" />.
         /// </summary>
         /// <param name="queue">Queue to store inbound reports in</param>
         /// <exception cref="ArgumentNullException">queueProvider;connectionFactory</exception>
-        public SaveReportHandler(IMessageQueue queue, IAdoNetUnitOfWork unitOfWork)
+        public SaveReportHandler(IMessageQueue queue, IAdoNetUnitOfWork unitOfWork, ConfigurationStore configStore)
         {
             _unitOfWork = unitOfWork;
             _queue = queue ?? throw new ArgumentNullException(nameof(queue));
+            var config= configStore.Load<ReportConfig>();
+            _maxSizeForJsonErrorReport = config?.MaxReportJsonSize ?? 1000000;
         }
 
         public void AddFilter(Func<NewReportDTO, bool> filter)
@@ -46,7 +49,8 @@ public void AddFilter(Func<NewReportDTO, bool> filter)
             _filters.Add(filter);
         }
 
-        public async Task BuildReportAsync(ClaimsPrincipal user, string appKey, string signatureProvidedByTheClient, string remoteAddress,
+        public async Task BuildReportAsync(ClaimsPrincipal user, string appKey, string signatureProvidedByTheClient,
+            string remoteAddress,
             byte[] reportBody)
         {
             if (!Guid.TryParse(appKey, out var tempKey))
@@ -70,6 +74,8 @@ public async Task BuildReportAsync(ClaimsPrincipal user, string appKey, string s
             }
 
             var report = DeserializeBody(reportBody);
+            if (report == null)
+                return;
 
             // correct incorrect clients
             if (report.CreatedAtUtc > DateTime.UtcNow)
@@ -130,6 +136,10 @@ private NewReportDTO DeserializeBody(byte[] body)
                 json = Encoding.UTF8.GetString(body);
             }
 
+            // protection against very large error reports.
+            if (json.Length > _maxSizeForJsonErrorReport)
+                return null;
+
             //to support clients that still use the OneTrueError client library.
             json = json.Replace("OneTrueError", "codeRR");
 
@@ -160,15 +170,14 @@ private async Task<AppInfo> GetAppAsync(string appKey)
                     };
                 }
             }
-
         }
 
         private async Task StoreInvalidReportAsync(string appKey, string sig, string remoteAddress, byte[] reportBody)
         {
             try
             {
                 //TODO: Make something generic.
-                using (var cmd = (SqlCommand)_unitOfWork.CreateCommand())
+                using (var cmd = (SqlCommand) _unitOfWork.CreateCommand())
                 {
                     cmd.CommandText =
                         @"INSERT INTO InvalidReports(appkey, signature, reportbody, errormessage, createdatutc)
@@ -185,7 +194,6 @@ private async Task StoreInvalidReportAsync(string appKey, string sig, string rem
                     cmd.AddParameter("createdatutc", DateTime.UtcNow);
                     await cmd.ExecuteNonQueryAsync();
                 }
-
             }
             catch (Exception ex)
             {
@@ -206,7 +214,7 @@ private Task StoreReportAsync(ClaimsPrincipal user, ProcessReport report)
             catch (Exception ex)
             {
                 _logger.Error(
-                    "Failed to StoreReport: " + JsonConvert.SerializeObject(new { model = report }), ex);
+                    "Failed to StoreReport: " + JsonConvert.SerializeObject(new {model = report}), ex);
             }
             return Task.FromResult<object>(null);
         }

src/Server/Coderr.Server.SqlServer/Core/Incidents/Queries/FindIncidentResultItemMapper.cs

@@ -24,6 +24,10 @@ public void Map(IDataRecord source, FindIncidentsResultItem destination)
             destination.IsReOpened = source["IsReopened"].Equals(1);
             destination.ReportCount = (int) source["ReportCount"];
             destination.LastUpdateAtUtc = (DateTime) source["UpdatedAtUtc"];
+
+            var value = source["LastReportAtUtc"];
+            destination.LastReportReceivedAtUtc = (DateTime) (value is DBNull ? destination.LastUpdateAtUtc : value);
+
             destination.CreatedAtUtc = (DateTime) source["CreatedAtUtc"];
         }
     }

src/Server/Coderr.Server.SqlServer/Core/Incidents/Queries/GetIncidentResultMapper.cs

@@ -25,6 +25,10 @@ public GetIncidentResultMapper()
             Property(x => x.Solution)
                 .ToPropertyValue(x => EntitySerializer.Deserialize<IncidentSolution>(x)?.Description);
 
+            Property(x => x.LastReportReceivedAtUtc)
+                .ToPropertyValue(DbConverters.ToEntityDate)
+                .ColumnName("LastReportAtUtc");
+
             Property(x => x.IsSolved).Ignore();
             Property(x => x.IsIgnored).Ignore();
 

src/Server/Coderr.Server.Web/Areas/Receiver/Controllers/ReportController.cs

@@ -8,6 +8,7 @@
 using System.Web;
 using System.Web.Http;
 using codeRR.Server.Infrastructure;
+using codeRR.Server.Infrastructure.Configuration;
 using codeRR.Server.ReportAnalyzer.Inbound;
 using codeRR.Server.Web.Areas.Receiver.Helpers;
 using codeRR.Server.Web.Areas.Receiver.Models;
@@ -24,15 +25,17 @@ public class ReportController : ApiController
         private static readonly SamplingCounter _samplingCounter = new SamplingCounter();
         private readonly ILog _logger = LogManager.GetLogger(typeof(ReportController));
         private readonly IMessageQueue _messageQueue;
+        private readonly ConfigurationStore _configStore;
 
         static ReportController()
         {
             _samplingCounter.Load();
         }
 
-        public ReportController(IMessageQueueProvider queueProvider, IAdoNetUnitOfWork unitOfWork)
+        public ReportController(IMessageQueueProvider queueProvider, IAdoNetUnitOfWork unitOfWork, ConfigurationStore configStore)
         {
             _unitOfWork = unitOfWork;
+            _configStore = configStore;
             _messageQueue = queueProvider.Open("Reports");
         }
 
@@ -51,7 +54,7 @@ public HttpResponseMessage Index()
         [HttpPost, Route("receiver/report/{appKey}")]
         public async Task<HttpResponseMessage> Post(string appKey, string sig)
         {
-            if (HttpContext.Current.Request.InputStream.Length > 20000000)
+            if (HttpContext.Current.Request.InputStream.Length > 2000000)
             {
                 return await KillLargeReportAsync(appKey);
             }
@@ -64,7 +67,7 @@ public async Task<HttpResponseMessage> Post(string appKey, string sig)
             {
                 var buffer = new byte[HttpContext.Current.Request.InputStream.Length];
                 HttpContext.Current.Request.InputStream.Read(buffer, 0, buffer.Length);
-                var handler = new SaveReportHandler(_messageQueue, _unitOfWork);
+                var handler = new SaveReportHandler(_messageQueue, _unitOfWork, _configStore);
                 await handler.BuildReportAsync(User as ClaimsPrincipal, appKey, sig, Request.GetClientIpAddress(), buffer);
                 return Request.CreateResponse(HttpStatusCode.OK);
             }

src/Server/Coderr.Server.Web/Scripts/Models/AllModels.js.map

@@ -762,19 +762,20 @@ module codeRR.Core.Incidents.Queries {
     export class GetIncidentResult {
         public static TYPE_NAME: string = 'GetIncidentResult';
         public ApplicationId: number;
+        public AssignedAtUtc: any;
+        public AssignedTo: string;
+        public AssignedToId: number;
         public ContextCollections: string[];
         public CreatedAtUtc: any;
         public DayStatistics: codeRR.Core.Incidents.Queries.ReportDay[];
         public Description: string;
         public FeedbackCount: number;
-        public AssignedToId: number;
         public FullName: string;
         public HashCodeIdentifier: string;
         public Id: number;
+        public IncidentState: number;
         public IsIgnored: boolean;
         public IsReOpened: boolean;
-        public AssignedTo: string;
-        public AssignedAtUtc: any;
         public IsSolutionShared: boolean;
         public IsSolved: boolean;
         public PreviousSolutionAtUtc: any;