Structured Logs: Buffering & flushing of logs

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Features
 
-- Add experimental support for capturing structured logs via `SentrySDK.logger` (#5532)
+- Add experimental support for capturing structured logs via `SentrySDK.logger` (#5532, #5593)
 
 ### Improvements
 

SentryTestUtils/TestClient.swift

@@ -150,8 +150,8 @@ public class TestClient: SentryClient {
         flushInvocations.record(timeout)
     }
 
-    public var captureLogsDataInvocations = Invocations<Data>()
-    public override func captureLogsData(_ data: Data) {
-        captureLogsDataInvocations.record(data)
+    public var captureLogsDataInvocations = Invocations<(data: Data, count: NSNumber)>()
+    public override func captureLogsData(_ data: Data, with count: NSNumber) {
+        captureLogsDataInvocations.record((data, count))
     }
 }

Sources/Sentry/SentryClient.m

@@ -1128,12 +1128,13 @@ - (void)removeAttachmentProcessor:(id<SentryClientAttachmentProcessor>)attachmen
     return processedAttachments;
 }
 
-- (void)captureLogsData:(NSData *)data
+- (void)captureLogsData:(NSData *)data with:(NSNumber *)itemCount;
 {
     SentryEnvelopeItemHeader *header =
         [[SentryEnvelopeItemHeader alloc] initWithType:SentryEnvelopeItemTypeLog
                                                 length:data.length
-                                           contentType:@"application/vnd.sentry.items.log+json"];
+                                           contentType:@"application/vnd.sentry.items.log+json"
+                                             itemCount:itemCount];
 
     SentryEnvelopeItem *envelopeItem = [[SentryEnvelopeItem alloc] initWithHeader:header data:data];
     SentryEnvelope *envelope = [[SentryEnvelope alloc] initWithHeader:[SentryEnvelopeHeader empty]

Sources/Sentry/SentrySDK.m

@@ -118,14 +118,19 @@ + (SentryReplayApi *)replay
 
 + (SentryLogger *)logger
 {
+
     @synchronized(currentLoggerLock) {
         if (currentLogger == nil) {
+
             SentryLogBatcher *batcher;
             if (nil != currentHub.client && currentHub.client.options.experimental.enableLogs) {
-                batcher = [[SentryLogBatcher alloc] initWithClient:currentHub.client];
+                batcher = [[SentryLogBatcher alloc]
+                    initWithClient:currentHub.client
+                     dispatchQueue:SentryDependencyContainer.sharedInstance.dispatchQueueWrapper];
             } else {
                 batcher = nil;
             }
+
             currentLogger = [[SentryLogger alloc]
                  initWithHub:currentHub
                 dateProvider:SentryDependencyContainer.sharedInstance.dateProvider

Sources/Sentry/include/SentryClient+Logs.h

@@ -7,7 +7,7 @@ NS_ASSUME_NONNULL_BEGIN
 /**
  * Helper to capture encoded logs, as SentryEnvelope can't be used in the Swift SDK.
  */
-- (void)captureLogsData:(NSData *)data;
+- (void)captureLogsData:(NSData *)data with:(NSNumber *)itemCount;
 
 @end
 

Sources/Swift/Tools/SentryLogBatcher.swift

@@ -6,24 +6,120 @@ import Foundation
 @_spi(Private) public class SentryLogBatcher: NSObject {
 
     private let client: SentryClient
+    private let flushTimeout: TimeInterval
+    private let maxBufferSizeBytes: Int
+    private let dispatchQueue: SentryDispatchQueueWrapper
 
-    @_spi(Private) public init(client: SentryClient) {
+    // All mutable state is accessed from the same dispatch queue.
+    private var encodedLogs: [Data] = []
+    private var encodedLogsSize: Int = 0
+    private var timerWorkItem: DispatchWorkItem?
+
+    @_spi(Private) public init(
+        client: SentryClient,
+        flushTimeout: TimeInterval,
+        maxBufferSizeBytes: Int,
+        dispatchQueue: SentryDispatchQueueWrapper
+    ) {
         self.client = client
+        self.flushTimeout = flushTimeout
+        self.maxBufferSizeBytes = maxBufferSizeBytes
+        self.dispatchQueue = dispatchQueue
         super.init()
     }
 
+    @_spi(Private) public convenience init(client: SentryClient, dispatchQueue: SentryDispatchQueueWrapper) {
+        self.init(
+            client: client,
+            flushTimeout: 5,
+            maxBufferSizeBytes: 1_024 * 1_024, // 1MB
+            dispatchQueue: dispatchQueue
+        )
+    }
+
     func add(_ log: SentryLog) {
-        dispatch(logs: [log])
+        dispatchQueue.dispatchAsync { [weak self] in
+            self?.encodeAndBuffer(log: log)
+        }
     }
 
-    private func dispatch(logs: [SentryLog]) {
+    @objc
+    public func flush() {
+        dispatchQueue.dispatchAsync { [weak self] in
+            self?.performFlush()
+        }
+    }
+
+    // Helper
+
+    // Only ever call this from the dispatch queue.
+    private func encodeAndBuffer(log: SentryLog) {
         do {
-            let payload = ["items": logs]
-            let data = try encodeToJSONData(data: payload)
+            let encodedLog = try encodeToJSONData(data: log)
+
+            let wasEmpty = encodedLogs.isEmpty
+
+            encodedLogs.append(encodedLog)
+            encodedLogsSize += encodedLog.count
+
+            let shouldFlushImmediatley = encodedLogsSize >= maxBufferSizeBytes
+            let shouldStartTimer = wasEmpty && timerWorkItem == nil && !shouldFlushImmediatley
 
-            client.captureLogsData(data)
+            if shouldStartTimer {
+                startTimer()
+            }
+            if shouldFlushImmediatley {
+                performFlush()
+            }
         } catch {
-            SentrySDKLog.error("Failed to create logs envelope.")
+            SentrySDKLog.error("Failed to encode log: \(error)")
+        }
+    }
+
+    // Only ever call this from the dispatch queue.
+    private func startTimer() {
+        let timerWorkItem = DispatchWorkItem { [weak self] in
+            self?.performFlush()
+        }
+        self.timerWorkItem = timerWorkItem
+        dispatchQueue.queue.asyncAfter(
+            deadline: .now() + flushTimeout,
+            execute: timerWorkItem
+        )
+    }
+
+    // Only ever call this from the dispatch queue.
+    private func performFlush() {
+        let encodedLogsToSend = Array(encodedLogs)
+
+        // Reset buffer state
+        encodedLogs.removeAll()
+        encodedLogsSize = 0
+
+        // Reset timer state
+        timerWorkItem?.cancel()
+        timerWorkItem = nil
+
+        // If there are no logs to send, return early.
+        guard encodedLogsToSend.count > 0 else {
+            return
+        }
+
+        // Create the payload.
+
+        var payloadData = Data()
+        payloadData.append(Data("{\"items\":[".utf8))
+        let separator = Data(",".utf8)
+        for (index, encodedLog) in encodedLogsToSend.enumerated() {
+            if index > 0 {
+                payloadData.append(separator)
+            }
+            payloadData.append(encodedLog)
         }
+        payloadData.append(Data("]}".utf8))
+
+        // Send the payload.
+
+        client.captureLogsData(payloadData, with: NSNumber(value: encodedLogsToSend.count))
     }
 }

Tests/SentryTests/SentryClientTests.swift

@@ -2098,7 +2098,7 @@ class SentryClientTest: XCTestCase {
         let sut = fixture.getSut()
         let logData = Data("{\"items\":[{\"timestamp\":1627846801,\"level\":\"info\",\"body\":\"Test log message\"}]}".utf8)
 
-        sut.captureLogsData(logData)
+        sut.captureLogsData(logData, with: NSNumber(value: 1))
 
         // Verify that an envelope was sent
         XCTAssertEqual(1, fixture.transport.sentEnvelopes.count)
@@ -2114,6 +2114,7 @@ class SentryClientTest: XCTestCase {
         XCTAssertEqual("log", item.header.type)
         XCTAssertEqual(UInt(logData.count), item.header.length)
         XCTAssertEqual("application/vnd.sentry.items.log+json", item.header.contentType)
+        XCTAssertEqual(NSNumber(value: 1), item.header.itemCount)
 
         // Verify the envelope item data
         XCTAssertEqual(logData, item.data)
@@ -2123,7 +2124,7 @@ class SentryClientTest: XCTestCase {
         let sut = fixture.getSutDisabledSdk()
         let logData = Data("{\"items\":[{\"timestamp\":1627846801,\"level\":\"info\",\"body\":\"Test log message\"}]}".utf8)
 
-        sut.captureLogsData(logData)
+        sut.captureLogsData(logData, with: NSNumber(value: 1))
 
         // Verify that no envelope was sent when client is disabled
         XCTAssertEqual(0, fixture.transport.sentEnvelopes.count)