Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Split up preload and nonpreload room state boxes
Browse files Browse the repository at this point in the history
krille-chan committed Dec 6, 2023
1 parent cf071e3 commit 4e80cc0
Showing 6 changed files with 322 additions and 183 deletions.
58 changes: 40 additions & 18 deletions lib/src/database/indexeddb_box.dart
Original file line number Diff line number Diff line change
@@ -13,7 +13,6 @@ class BoxCollection {
static Future<BoxCollection> open(
String name,
Set<String> boxNames, {
int version = 1,
Object? sqfliteDatabase,
IdbFactory? idbFactory,
}) async {
@@ -50,6 +49,11 @@ class BoxCollection {
if (cache.isEmpty) return;
final txn = _db.transaction(boxNames, readOnly ? 'readonly' : 'readwrite');
for (final fun in cache) {
// The IDB methods return a Future in Dart but must not be awaited in
// order to have an actual transaction. They must only be performed and
// then the transaction object must call `txn.completed;` which then
// returns the actual future.
// https://developer.mozilla.org/en-US/docs/Web/API/IDBTransaction
unawaited(fun(txn));
}
await txn.completed;
@@ -62,14 +66,24 @@ class BoxCollection {
}
}

Future<void> close() async => _db.close();
Future<void> close() async {
assert(_txnCache == null, 'Database closed while in transaction!');
return _db.close();
}
}

class Box<V> {
final String name;
final BoxCollection boxCollection;
final Map<String, V?> _cache = {};

/// _cachedKeys is only used to make sure that if you fetch all keys from a
/// box, you do not need to have an expensive read operation twice. There is
/// no other usage for this at the moment. So the cache is never partial.
/// Once the keys are cached, they need to be updated when changed in put and
/// delete* so that the cache does not become outdated.
Set<String>? _cachedKeys;

bool get _keysCached => _cachedKeys != null;

Box(this.name, this.boxCollection);
@@ -85,22 +99,13 @@ class Box<V> {
return keys;
}

Future<V?> getWhere(String indexName, String pattern,
[Transaction? txn]) async {
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final index = store.index(indexName);
final value = await index.get(pattern) as V?;
return value;
}

Future<Map<String, V>> getAllValues([Transaction? txn]) async {
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final map = <String, V>{};
final cursorStream = store.openCursor(autoAdvance: true);
await for (final cursor in cursorStream) {
map[cursor.key as String] = cursor.value as V;
map[cursor.key as String] = _fromValue(cursor.value) as V;
}
return map;
}
@@ -109,21 +114,22 @@ class Box<V> {
if (_cache.containsKey(key)) return _cache[key];
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
_cache[key] = await store.getObject(key) as V?;
_cache[key] = await store.getObject(key).then(_fromValue);
return _cache[key];
}

Future<List<V?>> getAll(List<String> keys, [Transaction? txn]) async {
if (!keys.any((key) => !_cache.containsKey(key))) {
if (keys.every((key) => _cache.containsKey(key))) {
return keys.map((key) => _cache[key]).toList();
}
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final list = await Future.wait(keys.map((key) => store.getObject(key)));
final list = await Future.wait(
keys.map((key) => store.getObject(key).then(_fromValue)));
for (var i = 0; i < keys.length; i++) {
_cache[keys[i]] = list[i] as V?;
_cache[keys[i]] = list[i];
}
return list.cast<V?>();
return list;
}

Future<void> put(String key, V val, [Transaction? txn]) async {
@@ -171,7 +177,7 @@ class Box<V> {
for (final key in keys) {
await store.delete(key);
_cache.remove(key);
_cachedKeys?.removeAll(keys);
_cachedKeys?.remove(key);
}
return;
}
@@ -191,4 +197,20 @@ class Box<V> {
_cachedKeys = null;
return;
}

V? _fromValue(Object? value) {
if (value == null) return null;
switch (V) {
case const (List<dynamic>):
return List.unmodifiable(value as List) as V;
case const (Map<dynamic, dynamic>):
return Map.unmodifiable(value as Map) as V;
case const (int):
case const (double):
case const (bool):
case const (String):
default:
return value as V;
}
}
}
321 changes: 188 additions & 133 deletions lib/src/database/matrix_sdk_database.dart

Large diffs are not rendered by default.

111 changes: 90 additions & 21 deletions lib/src/database/sqflite_box.dart
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import 'dart:async';
import 'dart:convert';

import 'package:sqflite_common/sqlite_api.dart';
@@ -13,7 +14,6 @@ class BoxCollection {
static Future<BoxCollection> open(
String name,
Set<String> boxNames, {
int version = 1,
Object? sqfliteDatabase,
dynamic idbFactory,
}) async {
@@ -40,19 +40,65 @@ class BoxCollection {

Batch? _activeBatch;

Completer<void>? _transactionLock;
final _transactionZones = <Zone>{};

Future<void> transaction(
Future<void> Function() action, {
List<String>? boxNames,
bool readOnly = false,
}) async {
boxNames ??= this.boxNames.toList();
_activeBatch = _db.batch();
await action();
final batch = _activeBatch;
_activeBatch = null;
if (batch == null) return;
await batch.commit(noResult: true);
return;
// we want transactions to lock, however NOT if transactoins are run inside of each other.
// to be able to do this, we use dart zones (https://dart.dev/articles/archive/zones).
// _transactionZones holds a set of all zones which are currently running a transaction.
// _transactionLock holds the lock.

// first we try to determine if we are inside of a transaction currently
var isInTransaction = false;
Zone? zone = Zone.current;
// for that we keep on iterating to the parent zone until there is either no zone anymore
// or we have found a zone inside of _transactionZones.
while (zone != null) {
if (_transactionZones.contains(zone)) {
isInTransaction = true;
break;
}
zone = zone.parent;
}
// if we are inside a transaction....just run the action
if (isInTransaction) {
return await action();
}
// if we are *not* in a transaction, time to wait for the lock!
while (_transactionLock != null) {
await _transactionLock!.future;
}
// claim the lock
final lock = Completer<void>();
_transactionLock = lock;
try {
// run the action inside of a new zone
return await runZoned(() async {
try {
// don't forget to add the new zone to _transactionZones!
_transactionZones.add(Zone.current);

final batch = _db.batch();
_activeBatch = batch;
await action();
_activeBatch = null;
await batch.commit(noResult: true);
return;
} finally {
// aaaand remove the zone from _transactionZones again
_transactionZones.remove(Zone.current);
}
});
} finally {
// aaaand finally release the lock
_transactionLock = null;
lock.complete();
}
}

Future<void> clear() => transaction(
@@ -70,6 +116,12 @@ class Box<V> {
final String name;
final BoxCollection boxCollection;
final Map<String, V?> _cache = {};

/// _cachedKeys is only used to make sure that if you fetch all keys from a
/// box, you do not need to have an expensive read operation twice. There is
/// no other usage for this at the moment. So the cache is never partial.
/// Once the keys are cached, they need to be updated when changed in put and
/// delete* so that the cache does not become outdated.
Set<String>? _cachedKeys;
bool get _keysCached => _cachedKeys != null;

@@ -119,8 +171,9 @@ class Box<V> {
case const (bool):
return (value == 'true') as V;
case const (List<dynamic>):
return List.unmodifiable(jsonDecode(value)) as V;
case const (Map<dynamic, dynamic>):
return jsonDecode(value) as V;
return Map.unmodifiable(jsonDecode(value)) as V;
case const (String):
default:
return value as V;
@@ -144,12 +197,12 @@ class Box<V> {

final result = await executor.query(name);
return Map.fromEntries(
result.where((row) => row['v'] != null).map(
(row) => MapEntry(
row['k'] as String,
_fromString(row['v']) as V,
),
),
result.map(
(row) => MapEntry(
row['k'] as String,
_fromString(row['v']) as V,
),
),
);
}

@@ -175,21 +228,37 @@ class Box<V> {
return keys.map((key) => _cache[key]).toList();
}

// The SQL operation might fail with more than 1000 keys. We define some
// buffer here and half the amount of keys recursively for this situation.
const getAllMax = 800;
if (keys.length > getAllMax) {
final half = keys.length ~/ 2;
return [
...(await getAll(keys.sublist(0, half))),
...(await getAll(keys.sublist(half))),
];
}

final executor = txn ?? boxCollection._db;

final list = <V?>[];

final result = await executor.query(
name,
where: 'k IN (${keys.map((_) => '?').join(',')})',
whereArgs: keys,
);
final resultMap = Map<String, V?>.fromEntries(result
.map((row) => MapEntry(row['k'] as String, _fromString(row['v']))));
final resultMap = Map<String, V?>.fromEntries(
result.map((row) => MapEntry(row['k'] as String, _fromString(row['v']))),
);

// We want to make sure that they values are returnd in the exact same
// order than the given keys. That's why we do this instead of just return
// `resultMap.values`.
list.addAll(keys.map((key) => resultMap[key]));

for (var i = 0; i < keys.length; i++) {
_cache[keys[i]] = list[i];
}
_cache.addAll(resultMap);

return list;
}

7 changes: 1 addition & 6 deletions test/box_test.dart
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
import 'dart:math';

import 'package:file/memory.dart';
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
import 'package:test/test.dart';

@@ -13,9 +10,7 @@ void main() {
const data = {'name': 'Fluffy', 'age': 2};
const data2 = {'name': 'Loki', 'age': 4};
setUp(() async {
final fileSystem = MemoryFileSystem();
final path = '${fileSystem.path}/${Random().nextDouble()}';
final db = await databaseFactoryFfi.openDatabase(path);
final db = await databaseFactoryFfi.openDatabase(':memory:');
collection = await BoxCollection.open(
'testbox',
boxNames,
4 changes: 2 additions & 2 deletions test/database_api_test.dart
Original file line number Diff line number Diff line change
@@ -27,10 +27,10 @@ import 'package:matrix/matrix.dart';
import 'fake_database.dart';

void main() {
group('HiveCollections Database Test', () {
group('Matrix SDK Database Test', () {
late DatabaseApi database;
late int toDeviceQueueIndex;
test('Open', () async {
test('Setup', () async {
database = await getMatrixSdkDatabase(null);
});
test('transaction', () async {
4 changes: 1 addition & 3 deletions test/fake_database.dart
Original file line number Diff line number Diff line change
@@ -43,9 +43,7 @@ Future<HiveCollectionsDatabase> getHiveCollectionsDatabase(Client? c) async {

// ignore: deprecated_member_use_from_same_package
Future<MatrixSdkDatabase> getMatrixSdkDatabase(Client? c) async {
final fileSystem = MemoryFileSystem();
final path = '${fileSystem.path}/${Random().nextDouble()}';
final database = await databaseFactoryFfi.openDatabase(path);
final database = await databaseFactoryFfi.openDatabase(':memory:');
final db = MatrixSdkDatabase('unit_test.${c?.hashCode}', database: database);
await db.open();
return db;

0 comments on commit 4e80cc0

Please sign in to comment.