feat: Store breadcrumbs to disk for OOM events

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- Store breadcrumbs to disk for OOM events (#2347)
+
 ### Fixes
 
 - Fix issue with invalid profiles uploading (#2358 and #2359)

Sentry.xcodeproj/project.pbxproj

@@ -40,6 +40,7 @@
 		0A1B497328E597DD00D7BFA3 /* TestLogOutput.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A1B497228E597DD00D7BFA3 /* TestLogOutput.swift */; };
 		0A1C3592287D7107007D01E3 /* SentryMetaTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A1C3591287D7107007D01E3 /* SentryMetaTests.swift */; };
 		0A2690B72885C2E000E4432D /* TestSentryPermissionsObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0AABE2EF2885C2120057ED69 /* TestSentryPermissionsObserver.swift */; };
+		0A2D7BBA29152CBF008727AF /* SentryOutOfMemoryScopeObserverTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A2D7BB929152CBF008727AF /* SentryOutOfMemoryScopeObserverTests.swift */; };
 		0A283E79291A67E000EF4126 /* SentryUIDeviceWrapperTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A283E78291A67E000EF4126 /* SentryUIDeviceWrapperTests.swift */; };
 		0A2D8D5B289815C0008720F6 /* SentryBaseIntegration.m in Sources */ = {isa = PBXBuildFile; fileRef = 0A2D8D5A289815C0008720F6 /* SentryBaseIntegration.m */; };
 		0A2D8D5D289815EB008720F6 /* SentryBaseIntegration.h in Headers */ = {isa = PBXBuildFile; fileRef = 0A2D8D5C289815EB008720F6 /* SentryBaseIntegration.h */; };
@@ -52,6 +53,8 @@
 		0A5370A128A3EC2400B2DCDE /* SentryViewHierarchyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A5370A028A3EC2400B2DCDE /* SentryViewHierarchyTests.swift */; };
 		0A56DA5F28ABA01B00C400D5 /* SentryTransactionContext+Private.h in Headers */ = {isa = PBXBuildFile; fileRef = 0A56DA5E28ABA01B00C400D5 /* SentryTransactionContext+Private.h */; };
 		0A6EEADD28A657970076B469 /* UIViewRecursiveDescriptionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A6EEADC28A657970076B469 /* UIViewRecursiveDescriptionTests.swift */; };
+		0A80E433291017C300095219 /* SentryOutOfMemoryScopeObserver.m in Sources */ = {isa = PBXBuildFile; fileRef = 0A80E432291017C300095219 /* SentryOutOfMemoryScopeObserver.m */; };
+		0A80E435291017D500095219 /* SentryOutOfMemoryScopeObserver.h in Headers */ = {isa = PBXBuildFile; fileRef = 0A80E434291017D500095219 /* SentryOutOfMemoryScopeObserver.h */; };
 		0A8F0A392886CC70000B15F6 /* SentryPermissionsObserver.h in Headers */ = {isa = PBXBuildFile; fileRef = 0AABE2EE288592750057ED69 /* SentryPermissionsObserver.h */; };
 		0A94158228F6C4C2006A5DD1 /* SentryAppStateManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A94158128F6C4C2006A5DD1 /* SentryAppStateManagerTests.swift */; };
 		0A9415BA28F96CAC006A5DD1 /* TestSentryReachability.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0A9415B928F96CAC006A5DD1 /* TestSentryReachability.swift */; };
@@ -767,6 +770,7 @@
 		03F9D37B2819A65C00602916 /* SentryProfilerTests.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = SentryProfilerTests.mm; sourceTree = "<group>"; };
 		0A1B497228E597DD00D7BFA3 /* TestLogOutput.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestLogOutput.swift; sourceTree = "<group>"; };
 		0A1C3591287D7107007D01E3 /* SentryMetaTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryMetaTests.swift; sourceTree = "<group>"; };
+		0A2D7BB929152CBF008727AF /* SentryOutOfMemoryScopeObserverTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryOutOfMemoryScopeObserverTests.swift; sourceTree = "<group>"; };
 		0A283E78291A67E000EF4126 /* SentryUIDeviceWrapperTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryUIDeviceWrapperTests.swift; sourceTree = "<group>"; };
 		0A2D8D5A289815C0008720F6 /* SentryBaseIntegration.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryBaseIntegration.m; sourceTree = "<group>"; };
 		0A2D8D5C289815EB008720F6 /* SentryBaseIntegration.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryBaseIntegration.h; path = include/SentryBaseIntegration.h; sourceTree = "<group>"; };
@@ -779,6 +783,8 @@
 		0A5370A028A3EC2400B2DCDE /* SentryViewHierarchyTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryViewHierarchyTests.swift; sourceTree = "<group>"; };
 		0A56DA5E28ABA01B00C400D5 /* SentryTransactionContext+Private.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "SentryTransactionContext+Private.h"; path = "include/SentryTransactionContext+Private.h"; sourceTree = "<group>"; };
 		0A6EEADC28A657970076B469 /* UIViewRecursiveDescriptionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIViewRecursiveDescriptionTests.swift; sourceTree = "<group>"; };
+		0A80E432291017C300095219 /* SentryOutOfMemoryScopeObserver.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryOutOfMemoryScopeObserver.m; sourceTree = "<group>"; };
+		0A80E434291017D500095219 /* SentryOutOfMemoryScopeObserver.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = SentryOutOfMemoryScopeObserver.h; path = include/SentryOutOfMemoryScopeObserver.h; sourceTree = "<group>"; };
 		0A94158128F6C4C2006A5DD1 /* SentryAppStateManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SentryAppStateManagerTests.swift; sourceTree = "<group>"; };
 		0A9415B928F96CAC006A5DD1 /* TestSentryReachability.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestSentryReachability.swift; sourceTree = "<group>"; };
 		0A9BF4E128A114940068D266 /* SentryViewHierarchyIntegration.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SentryViewHierarchyIntegration.m; sourceTree = "<group>"; };
@@ -2494,6 +2500,8 @@
 				7B6C5F8626034395007F7DFF /* SentryOutOfMemoryLogic.m */,
 				7B98D7CA25FB64EC00C5A389 /* SentryOutOfMemoryTrackingIntegration.h */,
 				7B98D7CE25FB650F00C5A389 /* SentryOutOfMemoryTrackingIntegration.m */,
+				0A80E434291017D500095219 /* SentryOutOfMemoryScopeObserver.h */,
+				0A80E432291017C300095219 /* SentryOutOfMemoryScopeObserver.m */,
 			);
 			name = OutOfMemory;
 			sourceTree = "<group>";
@@ -2616,6 +2624,7 @@
 			children = (
 				7B98D7DF25FB73B900C5A389 /* SentryOutOfMemoryTrackerTests.swift */,
 				7BFE7A0927A1B6B000D2B66E /* SentryOutOfMemoryIntegrationTests.swift */,
+				0A2D7BB929152CBF008727AF /* SentryOutOfMemoryScopeObserverTests.swift */,
 			);
 			path = OutOfMemory;
 			sourceTree = "<group>";
@@ -3185,6 +3194,7 @@
 				7B42C48027E08F33009B58C2 /* SentryDependencyContainer.h in Headers */,
 				6334314120AD9AE40077E581 /* SentryMechanism.h in Headers */,
 				03F84D2827DD414C008FE43F /* SentryCPU.h in Headers */,
+				0A80E435291017D500095219 /* SentryOutOfMemoryScopeObserver.h in Headers */,
 				7B610D642512399600B0B5D9 /* SentryHub+Private.h in Headers */,
 				D8F6A24B2885515C00320515 /* SentryPredicateDescriptor.h in Headers */,
 				639FCF9C1EBC7F9500778193 /* SentryThread.h in Headers */,
@@ -3405,6 +3415,7 @@
 				84A8891D28DBD28900C51DFD /* SentryDevice.mm in Sources */,
 				7B56D73324616D9500B842DA /* SentryConcurrentRateLimitsDictionary.m in Sources */,
 				8ECC674825C23A20000E2BF6 /* SentryTransaction.m in Sources */,
+				0A80E433291017C300095219 /* SentryOutOfMemoryScopeObserver.m in Sources */,
 				7BECF42826145CD900D9826E /* SentryMechanismMeta.m in Sources */,
 				8E7C982F2693D56000E6336C /* SentryTraceHeader.m in Sources */,
 				63FE715F20DA4C1100CDBAE8 /* SentryCrashID.c in Sources */,
@@ -3682,6 +3693,7 @@
 				7B30B68226527C55006B2752 /* TestDisplayLinkWrapper.swift in Sources */,
 				7BB6550D253EEB3900887E87 /* SentryUserFeedbackTests.swift in Sources */,
 				7BBD18B7245180FF00427C76 /* SentryDsnTests.m in Sources */,
+				0A2D7BBA29152CBF008727AF /* SentryOutOfMemoryScopeObserverTests.swift in Sources */,
 				7BD4BD4B27EB2DC20071F4FF /* SentryDiscardedEventTests.swift in Sources */,
 				63FE721A20DA66EC00CDBAE8 /* SentryCrashSysCtl_Tests.m in Sources */,
 				7B88F30424BC8E6500ADF90A /* SentrySerializationTests.swift in Sources */,

Sources/Sentry/SentryCrashScopeObserver.m

@@ -9,11 +9,6 @@
 #import <SentryScopeSyncC.h>
 #import <SentryUser.h>
 
-@interface
-SentryCrashScopeObserver ()
-
-@end
-
 @implementation SentryCrashScopeObserver
 
 - (instancetype)initWithMaxBreadcrumbs:(NSInteger)maxBreadcrumbs
@@ -90,10 +85,9 @@ - (void)setLevel:(enum SentryLevel)level
     sentrycrash_scopesync_setLevel([json bytes]);
 }
 
-- (void)addBreadcrumb:(SentryBreadcrumb *)crumb
+- (void)addSerializedBreadcrumb:(NSDictionary *)crumb
 {
-    NSDictionary *serialized = [crumb serialize];
-    NSData *json = [self toJSONEncodedCString:serialized];
+    NSData *json = [self toJSONEncodedCString:crumb];
     if (json == nil) {
         return;
     }

Sources/Sentry/SentryEvent.m

@@ -22,6 +22,14 @@
 
 @property (nonatomic) BOOL isCrashEvent;
 
+// We're storing serialized breadcrumbs to disk in JSON, and when we're reading them back (in
+// the case of OOM), we end up with the serialized breadcrumbs again. Instead of turning those
+// dictionaries into proper SentryBreadcrumb instances which then need to be serialized again in
+// SentryEvent, we use this serializedBreadcrumbs property to set the pre-serialized
+// breadcrumbs. It saves a LOT of work - especially turning an NSDictionary into a SentryBreadcrumb
+// is silly when we're just going to do the opposite right after.
+@property (nonatomic, strong) NSArray *serializedBreadcrumbs;
+
 @end
 
 @implementation SentryEvent
@@ -138,7 +146,13 @@ - (void)addSimpleProperties:(NSMutableDictionary *)serializedData
 
     [serializedData setValue:[self.stacktrace serialize] forKey:@"stacktrace"];
 
-    [serializedData setValue:[self serializeBreadcrumbs] forKey:@"breadcrumbs"];
+    NSMutableArray *breadcrumbs = [self serializeBreadcrumbs];
+    if (self.serializedBreadcrumbs.count > 0) {
+        [breadcrumbs addObjectsFromArray:self.serializedBreadcrumbs];
+    }
+    if (breadcrumbs.count > 0) {
+        [serializedData setValue:breadcrumbs forKey:@"breadcrumbs"];
+    }
 
     [serializedData setValue:[self.context sentry_sanitize] forKey:@"contexts"];
 
@@ -164,15 +178,12 @@ - (void)addSimpleProperties:(NSMutableDictionary *)serializedData
     }
 }
 
-- (NSArray *_Nullable)serializeBreadcrumbs
+- (NSMutableArray *)serializeBreadcrumbs
 {
     NSMutableArray *crumbs = [NSMutableArray new];
     for (SentryBreadcrumb *crumb in self.breadcrumbs) {
         [crumbs addObject:[crumb serialize]];
     }
-    if (crumbs.count <= 0) {
-        return nil;
-    }
     return crumbs;
 }
 

Sources/Sentry/SentryFileManager.m

@@ -28,6 +28,10 @@
 @property (nonatomic, copy) NSString *lastInForegroundFilePath;
 @property (nonatomic, copy) NSString *previousAppStateFilePath;
 @property (nonatomic, copy) NSString *appStateFilePath;
+@property (nonatomic, copy) NSString *previousBreadcrumbsFilePathOne;
+@property (nonatomic, copy) NSString *previousBreadcrumbsFilePathTwo;
+@property (nonatomic, copy) NSString *breadcrumbsFilePathOne;
+@property (nonatomic, copy) NSString *breadcrumbsFilePathTwo;
 @property (nonatomic, copy) NSString *timezoneOffsetFilePath;
 @property (nonatomic, assign) NSUInteger currentFileCounter;
 @property (nonatomic, assign) NSUInteger maxEnvelopes;
@@ -83,6 +87,14 @@ - (nullable instancetype)initWithOptions:(SentryOptions *)options
         self.previousAppStateFilePath =
             [self.sentryPath stringByAppendingPathComponent:@"previous.app.state"];
         self.appStateFilePath = [self.sentryPath stringByAppendingPathComponent:@"app.state"];
+        self.previousBreadcrumbsFilePathOne =
+            [self.sentryPath stringByAppendingPathComponent:@"previous.breadcrumbs.1.state"];
+        self.previousBreadcrumbsFilePathTwo =
+            [self.sentryPath stringByAppendingPathComponent:@"previous.breadcrumbs.2.state"];
+        self.breadcrumbsFilePathOne =
+            [self.sentryPath stringByAppendingPathComponent:@"breadcrumbs.1.state"];
+        self.breadcrumbsFilePathTwo =
+            [self.sentryPath stringByAppendingPathComponent:@"breadcrumbs.2.state"];
         self.timezoneOffsetFilePath =
             [self.sentryPath stringByAppendingPathComponent:@"timezone.offset"];
 
@@ -240,8 +252,12 @@ - (BOOL)removeFileAtPath:(NSString *)path
     NSError *error = nil;
     @synchronized(self) {
         [fileManager removeItemAtPath:path error:&error];
+
         if (nil != error) {
-            SENTRY_LOG_ERROR(@"Couldn't delete file %@: %@", path, error);
+            // We don't want to log an error if the file doesn't exist.
+            if (error.code != NSFileNoSuchFileError) {
+                SENTRY_LOG_ERROR(@"Couldn't delete file %@: %@", path, error);
+            }
             return NO;
         }
     }
@@ -455,26 +471,90 @@ - (void)storeAppState:(SentryAppState *)appState
 - (void)moveAppStateToPreviousAppState
 {
     @synchronized(self.appStateFilePath) {
-        NSFileManager *fileManager = [NSFileManager defaultManager];
+        [self moveState:self.appStateFilePath toPreviousState:self.previousAppStateFilePath];
+    }
+}
 
-        // We first need to remove the old previous app state file,
-        // or we can't move the current app state file to it.
-        [self removeFileAtPath:self.previousAppStateFilePath];
+- (void)moveBreadcrumbsToPreviousBreadcrumbs
+{
+    @synchronized(self.breadcrumbsFilePathOne) {
+        [self moveState:self.breadcrumbsFilePathOne
+            toPreviousState:self.previousBreadcrumbsFilePathOne];
+        [self moveState:self.breadcrumbsFilePathTwo
+            toPreviousState:self.previousBreadcrumbsFilePathTwo];
+    }
+}
 
-        NSError *error = nil;
-        [fileManager moveItemAtPath:self.appStateFilePath
-                             toPath:self.previousAppStateFilePath
-                              error:&error];
+- (void)moveState:(NSString *)stateFilePath toPreviousState:(NSString *)previousStateFilePath
+{
+    NSFileManager *fileManager = [NSFileManager defaultManager];
 
-        // We don't want to log an error if the file doesn't exist.
-        if (nil != error && error.code != NSFileNoSuchFileError) {
-            [SentryLog
-                logWithMessage:[NSString
-                                   stringWithFormat:
-                                       @"Failed to move app state to previous app state: %@", error]
-                      andLevel:kSentryLevelError];
+    // We first need to remove the old previous state file,
+    // or we can't move the current state file to it.
+    [self removeFileAtPath:previousStateFilePath];
+
+    NSError *error = nil;
+    [fileManager moveItemAtPath:stateFilePath toPath:previousStateFilePath error:&error];
+
+    // We don't want to log an error if the file doesn't exist.
+    if (nil != error && error.code != NSFileNoSuchFileError) {
+        SENTRY_LOG_ERROR(@"Failed to move %@ to previous state file: %@", stateFilePath, error);
+    }
+}
+
+- (NSArray *)readPreviousBreadcrumbs
+{
+    NSArray *fileOneLines = @[];
+    NSArray *fileTwoLines = @[];
+
+    if ([[NSFileManager defaultManager] fileExistsAtPath:self.previousBreadcrumbsFilePathOne]) {
+        NSString *fileContents =
+            [NSString stringWithContentsOfFile:self.previousBreadcrumbsFilePathOne
+                                      encoding:NSUTF8StringEncoding
+                                         error:nil];
+        fileOneLines = [fileContents
+            componentsSeparatedByCharactersInSet:[NSCharacterSet newlineCharacterSet]];
+    }
+
+    if ([[NSFileManager defaultManager] fileExistsAtPath:self.previousBreadcrumbsFilePathTwo]) {
+        NSString *fileContents =
+            [NSString stringWithContentsOfFile:self.previousBreadcrumbsFilePathTwo
+                                      encoding:NSUTF8StringEncoding
+                                         error:nil];
+        fileTwoLines = [fileContents
+            componentsSeparatedByCharactersInSet:[NSCharacterSet newlineCharacterSet]];
+    }
+
+    NSMutableArray *breadcrumbs = [NSMutableArray array];
+
+    if (fileOneLines.count > 0 || fileTwoLines.count > 0) {
+        NSArray *combinedLines;
+
+        if (fileOneLines.count > fileTwoLines.count) {
+            // If file one has more lines than file two, then file one contains the older crumbs,
+            // and thus needs to come first.
+            combinedLines = [fileOneLines arrayByAddingObjectsFromArray:fileTwoLines];
+        } else {
+            combinedLines = [fileTwoLines arrayByAddingObjectsFromArray:fileOneLines];
+        }
+
+        for (NSString *line in combinedLines) {
+            NSData *data = [line dataUsingEncoding:NSUTF8StringEncoding];
+
+            NSError *error;
+            NSDictionary *dict = [NSJSONSerialization JSONObjectWithData:data
+                                                                 options:0
+                                                                   error:&error];
+
+            if (error) {
+                SENTRY_LOG_ERROR(@"Error deserializing breadcrumb: %@", error);
+            } else {
+                [breadcrumbs addObject:dict];
+            }
         }
     }
+
+    return breadcrumbs;
 }
 
 - (SentryAppState *_Nullable)readAppState
@@ -494,8 +574,7 @@ - (SentryAppState *_Nullable)readPreviousAppState
 - (SentryAppState *_Nullable)readAppStateFrom:(NSString *)path
 {
     NSFileManager *fileManager = [NSFileManager defaultManager];
-    NSData *currentData = nil;
-    currentData = [fileManager contentsAtPath:path];
+    NSData *currentData = [fileManager contentsAtPath:path];
     if (nil == currentData) {
         return nil;
     }