diff --git a/lib/src/database/indexeddb_box.dart b/lib/src/database/indexeddb_box.dart index cf6561fb8..b67c1106d 100644 --- a/lib/src/database/indexeddb_box.dart +++ b/lib/src/database/indexeddb_box.dart @@ -13,7 +13,6 @@ class BoxCollection { static Future open( String name, Set boxNames, { - int version = 1, Object? sqfliteDatabase, IdbFactory? idbFactory, }) async { @@ -50,6 +49,11 @@ class BoxCollection { if (cache.isEmpty) return; final txn = _db.transaction(boxNames, readOnly ? 'readonly' : 'readwrite'); for (final fun in cache) { + // The IDB methods return a Future in Dart but must not be awaited in + // order to have an actual transaction. They must only be performed and + // then the transaction object must call `txn.completed;` which then + // returns the actual future. + // https://developer.mozilla.org/en-US/docs/Web/API/IDBTransaction unawaited(fun(txn)); } await txn.completed; @@ -62,14 +66,24 @@ class BoxCollection { } } - Future close() async => _db.close(); + Future close() async { + assert(_txnCache == null, 'Database closed while in transaction!'); + return _db.close(); + } } class Box { final String name; final BoxCollection boxCollection; final Map _cache = {}; + + /// _cachedKeys is only used to make sure that if you fetch all keys from a + /// box, you do not need to have an expensive read operation twice. There is + /// no other usage for this at the moment. So the cache is never partial. + /// Once the keys are cached, they need to be updated when changed in put and + /// delete* so that the cache does not become outdated. Set? _cachedKeys; + bool get _keysCached => _cachedKeys != null; Box(this.name, this.boxCollection); @@ -85,22 +99,13 @@ class Box { return keys; } - Future getWhere(String indexName, String pattern, - [Transaction? txn]) async { - txn ??= boxCollection._db.transaction(name, 'readonly'); - final store = txn.objectStore(name); - final index = store.index(indexName); - final value = await index.get(pattern) as V?; - return value; - } - Future> getAllValues([Transaction? txn]) async { txn ??= boxCollection._db.transaction(name, 'readonly'); final store = txn.objectStore(name); final map = {}; final cursorStream = store.openCursor(autoAdvance: true); await for (final cursor in cursorStream) { - map[cursor.key as String] = cursor.value as V; + map[cursor.key as String] = _fromValue(cursor.value) as V; } return map; } @@ -109,21 +114,22 @@ class Box { if (_cache.containsKey(key)) return _cache[key]; txn ??= boxCollection._db.transaction(name, 'readonly'); final store = txn.objectStore(name); - _cache[key] = await store.getObject(key) as V?; + _cache[key] = await store.getObject(key).then(_fromValue); return _cache[key]; } Future> getAll(List keys, [Transaction? txn]) async { - if (!keys.any((key) => !_cache.containsKey(key))) { + if (keys.every((key) => _cache.containsKey(key))) { return keys.map((key) => _cache[key]).toList(); } txn ??= boxCollection._db.transaction(name, 'readonly'); final store = txn.objectStore(name); - final list = await Future.wait(keys.map((key) => store.getObject(key))); + final list = await Future.wait( + keys.map((key) => store.getObject(key).then(_fromValue))); for (var i = 0; i < keys.length; i++) { - _cache[keys[i]] = list[i] as V?; + _cache[keys[i]] = list[i]; } - return list.cast(); + return list; } Future put(String key, V val, [Transaction? txn]) async { @@ -171,7 +177,7 @@ class Box { for (final key in keys) { await store.delete(key); _cache.remove(key); - _cachedKeys?.removeAll(keys); + _cachedKeys?.remove(key); } return; } @@ -191,4 +197,20 @@ class Box { _cachedKeys = null; return; } + + V? _fromValue(Object? value) { + if (value == null) return null; + switch (V) { + case const (List): + return List.unmodifiable(value as List) as V; + case const (Map): + return Map.unmodifiable(value as Map) as V; + case const (int): + case const (double): + case const (bool): + case const (String): + default: + return value as V; + } + } } diff --git a/lib/src/database/matrix_sdk_database.dart b/lib/src/database/matrix_sdk_database.dart index ffb9d4ee3..67c5e83fe 100644 --- a/lib/src/database/matrix_sdk_database.dart +++ b/lib/src/database/matrix_sdk_database.dart @@ -45,8 +45,14 @@ class MatrixSdkDatabase extends DatabaseApi { late Box _toDeviceQueueBox; /// Key is a tuple as TupleKey(roomId, type) where stateKey can be - /// an empty string. - late Box _roomStateBox; + /// an empty string. Must contain only states of type + /// client.importantRoomStates. + late Box _preloadRoomStateBox; + + /// Key is a tuple as TupleKey(roomId, type) where stateKey can be + /// an empty string. Must NOT contain states of a type from + /// client.importantRoomStates. + late Box _nonPreloadRoomStateBox; /// Key is a tuple as TupleKey(roomId, userId) late Box _roomMembersBox; @@ -86,43 +92,49 @@ class MatrixSdkDatabase extends DatabaseApi { final Directory? fileStoragePath; final Duration? deleteFilesAfterDuration; - String get _clientBoxName => 'box_client'; + static const String _clientBoxName = 'box_client'; + + static const String _accountDataBoxName = 'box_account_data'; - String get _accountDataBoxName => 'box_account_data'; + static const String _roomsBoxName = 'box_rooms'; - String get _roomsBoxName => 'box_rooms'; + static const String _toDeviceQueueBoxName = 'box_to_device_queue'; - String get _toDeviceQueueBoxName => 'box_to_device_queue'; + static const String _preloadRoomStateBoxName = 'box_preload_room_states'; - String get _roomStateBoxName => 'box_room_states'; + static const String _nonPreloadRoomStateBoxName = + 'box_non_preload_room_states'; - String get _roomMembersBoxName => 'box_room_members'; + static const String _roomMembersBoxName = 'box_room_members'; - String get _roomAccountDataBoxName => 'box_room_account_data'; + static const String _roomAccountDataBoxName = 'box_room_account_data'; - String get _inboundGroupSessionsBoxName => 'box_inbound_group_session'; + static const String _inboundGroupSessionsBoxName = + 'box_inbound_group_session'; - String get _outboundGroupSessionsBoxName => 'box_outbound_group_session'; + static const String _outboundGroupSessionsBoxName = + 'box_outbound_group_session'; - String get _olmSessionsBoxName => 'box_olm_session'; + static const String _olmSessionsBoxName = 'box_olm_session'; - String get _userDeviceKeysBoxName => 'box_user_device_keys'; + static const String _userDeviceKeysBoxName = 'box_user_device_keys'; - String get _userDeviceKeysOutdatedBoxName => 'box_user_device_keys_outdated'; + static const String _userDeviceKeysOutdatedBoxName = + 'box_user_device_keys_outdated'; - String get _userCrossSigningKeysBoxName => 'box_cross_signing_keys'; + static const String _userCrossSigningKeysBoxName = 'box_cross_signing_keys'; - String get _ssssCacheBoxName => 'box_ssss_cache'; + static const String _ssssCacheBoxName = 'box_ssss_cache'; - String get _presencesBoxName => 'box_presences'; + static const String _presencesBoxName = 'box_presences'; - String get _timelineFragmentsBoxName => 'box_timeline_fragments'; + static const String _timelineFragmentsBoxName = 'box_timeline_fragments'; - String get _eventsBoxName => 'box_events'; + static const String _eventsBoxName = 'box_events'; - String get _seenDeviceIdsBoxName => 'box_seen_device_ids'; + static const String _seenDeviceIdsBoxName = 'box_seen_device_ids'; - String get _seenDeviceKeysBoxName => 'box_seen_device_keys'; + static const String _seenDeviceKeysBoxName = 'box_seen_device_keys'; Database? database; @@ -148,7 +160,8 @@ class MatrixSdkDatabase extends DatabaseApi { _accountDataBoxName, _roomsBoxName, _toDeviceQueueBoxName, - _roomStateBoxName, + _preloadRoomStateBoxName, + _nonPreloadRoomStateBoxName, _roomMembersBoxName, _roomAccountDataBoxName, _inboundGroupSessionsBoxName, @@ -176,8 +189,11 @@ class MatrixSdkDatabase extends DatabaseApi { _roomsBox = _collection.openBox( _roomsBoxName, ); - _roomStateBox = _collection.openBox( - _roomStateBoxName, + _preloadRoomStateBox = _collection.openBox( + _preloadRoomStateBoxName, + ); + _nonPreloadRoomStateBox = _collection.openBox( + _nonPreloadRoomStateBoxName, ); _roomMembersBox = _collection.openBox( _roomMembersBoxName, @@ -249,7 +265,8 @@ class MatrixSdkDatabase extends DatabaseApi { Future clearCache() => transaction(() async { await _roomsBox.clear(); await _accountDataBox.clear(); - await _roomStateBox.clear(); + await _preloadRoomStateBox.clear(); + await _nonPreloadRoomStateBox.clear(); await _roomMembersBox.clear(); await _eventsBox.clear(); await _timelineFragmentsBox.clear(); @@ -291,34 +308,41 @@ class MatrixSdkDatabase extends DatabaseApi { } @override - Future forgetRoom(String roomId) => transaction(() async { - await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); - final eventsBoxKeys = await _eventsBox.getAllKeys(); - for (final key in eventsBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _eventsBox.delete(key); - } - final roomStateBoxKeys = await _roomStateBox.getAllKeys(); - for (final key in roomStateBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomStateBox.delete(key); - } - final roomMembersBoxKeys = await _roomMembersBox.getAllKeys(); - for (final key in roomMembersBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomMembersBox.delete(key); - } - final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys(); - for (final key in roomAccountDataBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomAccountDataBox.delete(key); - } - await _roomsBox.delete(roomId); - }); + Future forgetRoom(String roomId) async { + await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); + final eventsBoxKeys = await _eventsBox.getAllKeys(); + for (final key in eventsBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _eventsBox.delete(key); + } + final preloadRoomStateBoxKeys = await _preloadRoomStateBox.getAllKeys(); + for (final key in preloadRoomStateBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _preloadRoomStateBox.delete(key); + } + final nonPreloadRoomStateBoxKeys = + await _nonPreloadRoomStateBox.getAllKeys(); + for (final key in nonPreloadRoomStateBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _nonPreloadRoomStateBox.delete(key); + } + final roomMembersBoxKeys = await _roomMembersBox.getAllKeys(); + for (final key in roomMembersBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomMembersBox.delete(key); + } + final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys(); + for (final key in roomAccountDataBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomAccountDataBox.delete(key); + } + await _roomsBox.delete(roomId); + } @override Future> getAccountData() => @@ -329,7 +353,7 @@ class MatrixSdkDatabase extends DatabaseApi { for (final entry in raws.entries) { accountData[entry.key] = BasicEvent( type: entry.key, - content: copyMap(entry.value), + content: makeJson(entry.value), ); } return accountData; @@ -353,7 +377,7 @@ class MatrixSdkDatabase extends DatabaseApi { Future getEventById(String eventId, Room room) async { final raw = await _eventsBox.get(TupleKey(room.id, eventId).toString()); if (raw == null) return null; - return Event.fromJson(copyMap(raw), room); + return Event.fromJson(makeJson(raw), room); } /// Loads a whole list of events at once from the store for a specific room @@ -366,7 +390,7 @@ class MatrixSdkDatabase extends DatabaseApi { final rawEvents = await _eventsBox.getAll(keys); return rawEvents .whereType() - .map((rawEvent) => Event.fromJson(copyMap(rawEvent), room)) + .map((rawEvent) => Event.fromJson(makeJson(rawEvent), room)) .toList(); } @@ -423,7 +447,7 @@ class MatrixSdkDatabase extends DatabaseApi { ) async { final raw = await _inboundGroupSessionsBox.get(sessionId); if (raw == null) return null; - return StoredInboundGroupSession.fromJson(copyMap(raw)); + return StoredInboundGroupSession.fromJson(makeJson(raw)); } @override @@ -435,7 +459,7 @@ class MatrixSdkDatabase extends DatabaseApi { .take(50) .map( (json) => StoredInboundGroupSession.fromJson( - copyMap(json), + makeJson(json), ), ) .toList(); @@ -454,7 +478,8 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future storeOlmSession(String identityKey, String sessionId, String pickle, int lastReceived) async { - final rawSessions = copyMap((await _olmSessionsBox.get(identityKey)) ?? {}); + final rawSessions = + makeJson((await _olmSessionsBox.get(identityKey)) ?? {}, true); rawSessions[sessionId] = { 'identity_key': identityKey, 'pickle': pickle, @@ -471,7 +496,7 @@ class MatrixSdkDatabase extends DatabaseApi { final rawSessions = await _olmSessionsBox.get(identityKey); if (rawSessions == null || rawSessions.isEmpty) return []; return rawSessions.values - .map((json) => OlmSession.fromJson(copyMap(json), userId)) + .map((json) => OlmSession.fromJson(makeJson(json), userId)) .toList(); } @@ -492,7 +517,7 @@ class MatrixSdkDatabase extends DatabaseApi { String roomId, String userId) async { final raw = await _outboundGroupSessionsBox.get(roomId); if (raw == null) return null; - return OutboundGroupSession.fromJson(copyMap(raw), userId); + return OutboundGroupSession.fromJson(makeJson(raw), userId); } @override @@ -501,17 +526,17 @@ class MatrixSdkDatabase extends DatabaseApi { // Get raw room from database: final roomData = await _roomsBox.get(roomId); if (roomData == null) return null; - final room = Room.fromJson(copyMap(roomData), client); + final room = Room.fromJson(makeJson(roomData), client); // Get important states: if (loadImportantStates) { final dbKeys = client.importantStateEvents .map((state) => TupleKey(roomId, state).toString()) .toList(); - final rawStates = await _roomStateBox.getAll(dbKeys); + final rawStates = await _preloadRoomStateBox.getAll(dbKeys); for (final rawState in rawStates) { if (rawState == null || rawState[''] == null) continue; - room.setState(Event.fromJson(copyMap(rawState['']), room)); + room.setState(Event.fromJson(makeJson(rawState['']), room)); } } @@ -525,35 +550,29 @@ class MatrixSdkDatabase extends DatabaseApi { final rawRooms = await _roomsBox.getAllValues(); - final getRoomStateRequests = >{}; - for (final raw in rawRooms.values) { // Get the room - final room = Room.fromJson(copyMap(raw), client); - // Get the "important" room states. All other states will be loaded once - // `getUnimportantRoomStates()` is called. - final dbKeys = client.importantStateEvents - .map((state) => TupleKey(room.id, state).toString()) - .toList(); - getRoomStateRequests[room.id] = _roomStateBox.getAll(dbKeys); + final room = Room.fromJson(makeJson(raw), client); // Add to the list and continue. rooms[room.id] = room; } - for (final room in rooms.values) { - // Add states to the room - final statesList = await getRoomStateRequests[room.id]; - if (statesList != null) { - for (final states in statesList) { - if (states == null) continue; - final stateEvents = states.values - .map((raw) => Event.fromJson(copyMap(raw), room)) - .toList(); - for (final state in stateEvents) { - room.setState(state); - } - } + final roomStatesDataRaws = await _preloadRoomStateBox.getAllValues(); + for (final entry in roomStatesDataRaws.entries) { + final keys = TupleKey.fromString(entry.key); + final roomId = keys.parts.first; + final room = rooms[roomId]; + if (room == null) { + Logs().w('Found event in store for unknown room', entry.value); + continue; + } + final states = entry.value; + final stateEvents = states.values + .map((raw) => Event.fromJson(makeJson(raw), room)) + .toList(); + for (final state in stateEvents) { + room.setState(state); } } @@ -562,7 +581,7 @@ class MatrixSdkDatabase extends DatabaseApi { for (final entry in roomAccountDataRaws.entries) { final keys = TupleKey.fromString(entry.key); final basicRoomEvent = BasicRoomEvent.fromJson( - copyMap(entry.value), + makeJson(entry.value), ); final roomId = keys.parts.first; if (rooms.containsKey(roomId)) { @@ -583,16 +602,16 @@ class MatrixSdkDatabase extends DatabaseApi { Future getSSSSCache(String type) async { final raw = await _ssssCacheBox.get(type); if (raw == null) return null; - return SSSSCache.fromJson(copyMap(raw)); + return SSSSCache.fromJson(makeJson(raw)); } @override Future> getToDeviceEventQueue() async { final raws = await _toDeviceQueueBox.getAllValues(); final copiedRaws = raws.entries.map((entry) { - final copiedRaw = copyMap(entry.value); + final copiedRaw = makeJson(entry.value, true); copiedRaw['id'] = int.parse(entry.key); - copiedRaw['content'] = jsonDecode(copiedRaw['content']); + copiedRaw['content'] = jsonDecode(copiedRaw['content'] as String); return copiedRaw; }).toList(); return copiedRaws.map((raw) => QueuedToDeviceEvent.fromJson(raw)).toList(); @@ -601,17 +620,17 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future> getUnimportantRoomEventStatesForRoom( List events, Room room) async { - final keys = (await _roomStateBox.getAllKeys()).where((key) { + final keys = (await _nonPreloadRoomStateBox.getAllKeys()).where((key) { final tuple = TupleKey.fromString(key); return tuple.parts.first == room.id && !events.contains(tuple.parts[1]); }); final unimportantEvents = []; for (final key in keys) { - final states = await _roomStateBox.get(key); + final states = await _nonPreloadRoomStateBox.get(key); if (states == null) continue; unimportantEvents.addAll( - states.values.map((raw) => Event.fromJson(copyMap(raw), room))); + states.values.map((raw) => Event.fromJson(makeJson(raw), room))); } return unimportantEvents; } @@ -621,7 +640,7 @@ class MatrixSdkDatabase extends DatabaseApi { final state = await _roomMembersBox.get(TupleKey(room.id, userId).toString()); if (state == null) return null; - return Event.fromJson(copyMap(state), room).asUser; + return Event.fromJson(makeJson(state), room).asUser; } @override @@ -651,14 +670,14 @@ class MatrixSdkDatabase extends DatabaseApi { (key) { final userDeviceKey = userDeviceKeys[key]; if (userDeviceKey == null) return null; - return copyMap(userDeviceKey); + return makeJson(userDeviceKey); }, ); final crossSigningEntries = crossSigningKeysBoxKeys.map( (key) { final crossSigningKey = userCrossSigningKeys[key]; if (crossSigningKey == null) return null; - return copyMap(crossSigningKey); + return makeJson(crossSigningKey); }, ); res[userId] = DeviceKeysList.fromDbJson( @@ -689,7 +708,7 @@ class MatrixSdkDatabase extends DatabaseApi { final states = await _roomMembersBox.getAll(keys); states.removeWhere((state) => state == null); for (final state in states) { - users.add(Event.fromJson(copyMap(state!), room).asUser); + users.add(Event.fromJson(makeJson(state!), room).asUser); } return users; @@ -749,7 +768,10 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future markInboundGroupSessionAsUploaded( String roomId, String sessionId) async { - final raw = copyMap(await _inboundGroupSessionsBox.get(sessionId) ?? {}); + final raw = makeJson( + await _inboundGroupSessionsBox.get(sessionId) ?? {}, + true, + ); if (raw.isEmpty) { Logs().w( 'Tried to mark inbound group session as uploaded which was not found in the database!'); @@ -764,7 +786,10 @@ class MatrixSdkDatabase extends DatabaseApi { Future markInboundGroupSessionsAsNeedingUpload() async { final keys = await _inboundGroupSessionsBox.getAllKeys(); for (final sessionId in keys) { - final raw = copyMap(await _inboundGroupSessionsBox.get(sessionId) ?? {}); + final raw = makeJson( + await _inboundGroupSessionsBox.get(sessionId) ?? {}, + true, + ); if (raw.isEmpty) continue; raw['uploaded'] = false; await _inboundGroupSessionsBox.put(sessionId, raw); @@ -812,9 +837,12 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setBlockedUserCrossSigningKey( bool blocked, String userId, String publicKey) async { - final raw = copyMap(await _userCrossSigningKeysBox - .get(TupleKey(userId, publicKey).toString()) ?? - {}); + final raw = makeJson( + await _userCrossSigningKeysBox + .get(TupleKey(userId, publicKey).toString()) ?? + {}, + true, + ); raw['blocked'] = blocked; await _userCrossSigningKeysBox.put( TupleKey(userId, publicKey).toString(), @@ -826,9 +854,10 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setBlockedUserDeviceKey( bool blocked, String userId, String deviceId) async { - final raw = copyMap( + final raw = makeJson( await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? - {}); + {}, + true); raw['blocked'] = blocked; await _userDeviceKeysBox.put( TupleKey(userId, deviceId).toString(), @@ -840,9 +869,10 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setLastActiveUserDeviceKey( int lastActive, String userId, String deviceId) async { - final raw = copyMap( + final raw = makeJson( await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? - {}); + {}, + true); raw['last_active'] = lastActive; await _userDeviceKeysBox.put( @@ -854,9 +884,10 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setLastSentMessageUserDeviceKey( String lastSentMessage, String userId, String deviceId) async { - final raw = copyMap( - await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? - {}); + final raw = makeJson( + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? {}, + true, + ); raw['last_sent_message'] = lastSentMessage; await _userDeviceKeysBox.put( TupleKey(userId, deviceId).toString(), @@ -869,7 +900,7 @@ class MatrixSdkDatabase extends DatabaseApi { String? prevBatch, String roomId, Client client) async { final raw = await _roomsBox.get(roomId); if (raw == null) return; - final room = Room.fromJson(copyMap(raw), client); + final room = Room.fromJson(makeJson(raw), client); room.prev_batch = prevBatch; await _roomsBox.put(roomId, room.toJson()); return; @@ -878,9 +909,12 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setVerifiedUserCrossSigningKey( bool verified, String userId, String publicKey) async { - final raw = copyMap((await _userCrossSigningKeysBox - .get(TupleKey(userId, publicKey).toString())) ?? - {}); + final raw = makeJson( + (await _userCrossSigningKeysBox + .get(TupleKey(userId, publicKey).toString())) ?? + {}, + true, + ); raw['verified'] = verified; await _userCrossSigningKeysBox.put( TupleKey(userId, publicKey).toString(), @@ -892,9 +926,10 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future setVerifiedUserDeviceKey( bool verified, String userId, String deviceId) async { - final raw = copyMap( - await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? - {}); + final raw = makeJson( + await _userDeviceKeysBox.get(TupleKey(userId, deviceId).toString()) ?? {}, + true, + ); raw['verified'] = verified; await _userDeviceKeysBox.put( TupleKey(userId, deviceId).toString(), @@ -905,7 +940,7 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future storeAccountData(String type, String content) async { - await _accountDataBox.put(type, copyMap(jsonDecode(content))); + await _accountDataBox.put(type, makeJson(jsonDecode(content))); return; } @@ -928,10 +963,17 @@ class MatrixSdkDatabase extends DatabaseApi { event.toJson()); if (tmpRoom.lastEvent?.eventId == event.eventId) { - await _roomStateBox.put( - TupleKey(eventUpdate.roomID, event.type).toString(), - {'': event.toJson()}, - ); + if (client.importantStateEvents.contains(event.type)) { + await _preloadRoomStateBox.put( + TupleKey(eventUpdate.roomID, event.type).toString(), + {'': event.toJson()}, + ); + } else { + await _nonPreloadRoomStateBox.put( + TupleKey(eventUpdate.roomID, event.type).toString(), + {'': event.toJson()}, + ); + } } } } @@ -946,7 +988,7 @@ class MatrixSdkDatabase extends DatabaseApi { final prevStatus = prevEvent == null ? null : () { - final json = copyMap(prevEvent); + final json = makeJson(prevEvent); final statusInt = json.tryGet('status') ?? json .tryGetMap('unsigned') @@ -1047,11 +1089,15 @@ class MatrixSdkDatabase extends DatabaseApi { ).toString(), eventUpdate.content); } else { + final type = eventUpdate.content['type'] as String; + final roomStateBox = client.importantStateEvents.contains(type) + ? _preloadRoomStateBox + : _nonPreloadRoomStateBox; final key = TupleKey( eventUpdate.roomID, - eventUpdate.content['type'], + type, ).toString(); - final stateMap = copyMap(await _roomStateBox.get(key) ?? {}); + final stateMap = makeJson(await roomStateBox.get(key) ?? {}, true); // store state events and new messages, that either are not an edit or an edit of the lastest message // An edit is an event, that has an edit relation to the latest event. In some cases for the second edit, we need to compare if both have an edit relation to the same event instead. if (eventUpdate.content @@ -1059,7 +1105,7 @@ class MatrixSdkDatabase extends DatabaseApi { ?.tryGetMap('m.relates_to') == null) { stateMap[stateKey] = eventUpdate.content; - await _roomStateBox.put(key, stateMap); + await roomStateBox.put(key, stateMap); } else { final editedEventRelationshipEventId = eventUpdate.content .tryGetMap('content') @@ -1088,7 +1134,7 @@ class MatrixSdkDatabase extends DatabaseApi { ?.relationshipEventId) // edit of latest (edited event) event ) { stateMap[stateKey] = eventUpdate.content; - await _roomStateBox.put(key, stateMap); + await roomStateBox.put(key, stateMap); } } } @@ -1204,7 +1250,7 @@ class MatrixSdkDatabase extends DatabaseApi { membership: membership, ).toJson()); } else if (roomUpdate is JoinedRoomUpdate) { - final currentRoom = Room.fromJson(copyMap(currentRawRoom), client); + final currentRoom = Room.fromJson(makeJson(currentRawRoom), client); await _roomsBox.put( roomId, Room( @@ -1361,7 +1407,7 @@ class MatrixSdkDatabase extends DatabaseApi { 'Tried to update inbound group session indexes of a session which was not found in the database!'); return; } - final json = copyMap(raw); + final json = makeJson(raw, true); json['indexes'] = indexes; await _inboundGroupSessionsBox.put(sessionId, json); return; @@ -1371,7 +1417,7 @@ class MatrixSdkDatabase extends DatabaseApi { Future> getAllInboundGroupSessions() async { final rawSessions = await _inboundGroupSessionsBox.getAllValues(); return rawSessions.values - .map((raw) => StoredInboundGroupSession.fromJson(copyMap(raw))) + .map((raw) => StoredInboundGroupSession.fromJson(makeJson(raw))) .toList(); } @@ -1411,7 +1457,8 @@ class MatrixSdkDatabase extends DatabaseApi { _clientBoxName: await _clientBox.getAllValues(), _accountDataBoxName: await _accountDataBox.getAllValues(), _roomsBoxName: await _roomsBox.getAllValues(), - _roomStateBoxName: await _roomStateBox.getAllValues(), + _preloadRoomStateBoxName: await _preloadRoomStateBox.getAllValues(), + _nonPreloadRoomStateBoxName: await _nonPreloadRoomStateBox.getAllValues(), _roomMembersBoxName: await _roomMembersBox.getAllValues(), _toDeviceQueueBoxName: await _toDeviceQueueBox.getAllValues(), _roomAccountDataBoxName: await _roomAccountDataBox.getAllValues(), @@ -1452,8 +1499,13 @@ class MatrixSdkDatabase extends DatabaseApi { for (final key in json[_roomsBoxName]!.keys) { await _roomsBox.put(key, json[_roomsBoxName]![key]); } - for (final key in json[_roomStateBoxName]!.keys) { - await _roomStateBox.put(key, json[_roomStateBoxName]![key]); + for (final key in json[_preloadRoomStateBoxName]!.keys) { + await _preloadRoomStateBox.put( + key, json[_preloadRoomStateBoxName]![key]); + } + for (final key in json[_nonPreloadRoomStateBoxName]!.keys) { + await _nonPreloadRoomStateBox.put( + key, json[_nonPreloadRoomStateBoxName]![key]); } for (final key in json[_roomMembersBoxName]!.keys) { await _roomMembersBox.put(key, json[_roomMembersBoxName]![key]); @@ -1550,6 +1602,9 @@ class MatrixSdkDatabase extends DatabaseApi { final rawPresence = await _presencesBox.get(userId); if (rawPresence == null) return null; - return CachedPresence.fromJson(copyMap(rawPresence)); + return CachedPresence.fromJson(makeJson(rawPresence)); } } + +Map makeJson(Map json, [bool copy = false]) => + copy ? copyMap(json) : json.cast(); diff --git a/lib/src/database/sqflite_box.dart b/lib/src/database/sqflite_box.dart index 9b4a6e914..21053a38c 100644 --- a/lib/src/database/sqflite_box.dart +++ b/lib/src/database/sqflite_box.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'package:sqflite_common/sqlite_api.dart'; @@ -13,7 +14,6 @@ class BoxCollection { static Future open( String name, Set boxNames, { - int version = 1, Object? sqfliteDatabase, dynamic idbFactory, }) async { @@ -40,19 +40,65 @@ class BoxCollection { Batch? _activeBatch; + Completer? _transactionLock; + final _transactionZones = {}; + Future transaction( Future Function() action, { List? boxNames, bool readOnly = false, }) async { - boxNames ??= this.boxNames.toList(); - _activeBatch = _db.batch(); - await action(); - final batch = _activeBatch; - _activeBatch = null; - if (batch == null) return; - await batch.commit(noResult: true); - return; + // we want transactions to lock, however NOT if transactoins are run inside of each other. + // to be able to do this, we use dart zones (https://dart.dev/articles/archive/zones). + // _transactionZones holds a set of all zones which are currently running a transaction. + // _transactionLock holds the lock. + + // first we try to determine if we are inside of a transaction currently + var isInTransaction = false; + Zone? zone = Zone.current; + // for that we keep on iterating to the parent zone until there is either no zone anymore + // or we have found a zone inside of _transactionZones. + while (zone != null) { + if (_transactionZones.contains(zone)) { + isInTransaction = true; + break; + } + zone = zone.parent; + } + // if we are inside a transaction....just run the action + if (isInTransaction) { + return await action(); + } + // if we are *not* in a transaction, time to wait for the lock! + while (_transactionLock != null) { + await _transactionLock!.future; + } + // claim the lock + final lock = Completer(); + _transactionLock = lock; + try { + // run the action inside of a new zone + return await runZoned(() async { + try { + // don't forget to add the new zone to _transactionZones! + _transactionZones.add(Zone.current); + + final batch = _db.batch(); + _activeBatch = batch; + await action(); + _activeBatch = null; + await batch.commit(noResult: true); + return; + } finally { + // aaaand remove the zone from _transactionZones again + _transactionZones.remove(Zone.current); + } + }); + } finally { + // aaaand finally release the lock + _transactionLock = null; + lock.complete(); + } } Future clear() => transaction( @@ -70,6 +116,12 @@ class Box { final String name; final BoxCollection boxCollection; final Map _cache = {}; + + /// _cachedKeys is only used to make sure that if you fetch all keys from a + /// box, you do not need to have an expensive read operation twice. There is + /// no other usage for this at the moment. So the cache is never partial. + /// Once the keys are cached, they need to be updated when changed in put and + /// delete* so that the cache does not become outdated. Set? _cachedKeys; bool get _keysCached => _cachedKeys != null; @@ -119,8 +171,9 @@ class Box { case const (bool): return (value == 'true') as V; case const (List): + return List.unmodifiable(jsonDecode(value)) as V; case const (Map): - return jsonDecode(value) as V; + return Map.unmodifiable(jsonDecode(value)) as V; case const (String): default: return value as V; @@ -144,12 +197,12 @@ class Box { final result = await executor.query(name); return Map.fromEntries( - result.where((row) => row['v'] != null).map( - (row) => MapEntry( - row['k'] as String, - _fromString(row['v']) as V, - ), - ), + result.map( + (row) => MapEntry( + row['k'] as String, + _fromString(row['v']) as V, + ), + ), ); } @@ -175,21 +228,37 @@ class Box { return keys.map((key) => _cache[key]).toList(); } + // The SQL operation might fail with more than 1000 keys. We define some + // buffer here and half the amount of keys recursively for this situation. + const getAllMax = 800; + if (keys.length > getAllMax) { + final half = keys.length ~/ 2; + return [ + ...(await getAll(keys.sublist(0, half))), + ...(await getAll(keys.sublist(half))), + ]; + } + final executor = txn ?? boxCollection._db; final list = []; + final result = await executor.query( name, where: 'k IN (${keys.map((_) => '?').join(',')})', whereArgs: keys, ); - final resultMap = Map.fromEntries(result - .map((row) => MapEntry(row['k'] as String, _fromString(row['v'])))); + final resultMap = Map.fromEntries( + result.map((row) => MapEntry(row['k'] as String, _fromString(row['v']))), + ); + + // We want to make sure that they values are returnd in the exact same + // order than the given keys. That's why we do this instead of just return + // `resultMap.values`. list.addAll(keys.map((key) => resultMap[key])); - for (var i = 0; i < keys.length; i++) { - _cache[keys[i]] = list[i]; - } + _cache.addAll(resultMap); + return list; } diff --git a/test/box_test.dart b/test/box_test.dart index 187ee2928..d58cf6167 100644 --- a/test/box_test.dart +++ b/test/box_test.dart @@ -1,6 +1,3 @@ -import 'dart:math'; - -import 'package:file/memory.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:test/test.dart'; @@ -13,9 +10,7 @@ void main() { const data = {'name': 'Fluffy', 'age': 2}; const data2 = {'name': 'Loki', 'age': 4}; setUp(() async { - final fileSystem = MemoryFileSystem(); - final path = '${fileSystem.path}/${Random().nextDouble()}'; - final db = await databaseFactoryFfi.openDatabase(path); + final db = await databaseFactoryFfi.openDatabase(':memory:'); collection = await BoxCollection.open( 'testbox', boxNames, diff --git a/test/database_api_test.dart b/test/database_api_test.dart index f85d97013..6e3c9999e 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -27,10 +27,10 @@ import 'package:matrix/matrix.dart'; import 'fake_database.dart'; void main() { - group('HiveCollections Database Test', () { + group('Matrix SDK Database Test', () { late DatabaseApi database; late int toDeviceQueueIndex; - test('Open', () async { + test('Setup', () async { database = await getMatrixSdkDatabase(null); }); test('transaction', () async { diff --git a/test/fake_database.dart b/test/fake_database.dart index 59d966834..af428557d 100644 --- a/test/fake_database.dart +++ b/test/fake_database.dart @@ -43,9 +43,7 @@ Future getHiveCollectionsDatabase(Client? c) async { // ignore: deprecated_member_use_from_same_package Future getMatrixSdkDatabase(Client? c) async { - final fileSystem = MemoryFileSystem(); - final path = '${fileSystem.path}/${Random().nextDouble()}'; - final database = await databaseFactoryFfi.openDatabase(path); + final database = await databaseFactoryFfi.openDatabase(':memory:'); final db = MatrixSdkDatabase('unit_test.${c?.hashCode}', database: database); await db.open(); return db;