Skip to content

Commit fb71c04

Browse files
Merge branch '121-fix-ios-vector-search' into 'main'
Use visitor for Query.findWithScores, update to C API 4.0.1 See merge request objectbox/objectbox-dart!94
2 parents 392bb30 + f7b7a33 commit fb71c04

21 files changed

+347
-70
lines changed

dev-doc/updating-c-library.md

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,17 @@ For Dart Native and unit tests ([install.sh](../install.sh)),
99
for the binding update script (see below) and
1010
for Flutter (`flutter_libs` and `sync_flutter_libs` plugins) on Linux and Windows:
1111
```
12-
./tool/set-c-version.sh 4.0.0
12+
./tool/set-c-version.sh 4.0.1
1313
```
1414

1515
For the Flutter plugins on Android ([view releases](https://github.com/objectbox/objectbox-java/releases)):
1616
```
17-
./tool/set-android-version.sh 4.0.0
17+
./tool/set-android-version.sh 4.0.2
1818
```
1919

2020
For the Flutter plugins on iOS/macOS ([view releases](https://github.com/objectbox/objectbox-swift/releases))
2121
```
22-
./tool/set-swift-version.sh 2.0.0
22+
./tool/set-swift-version.sh 4.0.0
2323
```
2424

2525
For each, add an entry (see previous releases) to the [CHANGELOG](../objectbox/CHANGELOG.md).

flutter_libs/android/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,6 @@ android {
5252
// ObjectBox Android library that includes an ObjectBox C library version compatible with
5353
// the C API binding of the ObjectBox Dart package.
5454
// https://central.sonatype.com/search?q=g:io.objectbox%20objectbox-android
55-
implementation "io.objectbox:objectbox-android:4.0.0"
55+
implementation "io.objectbox:objectbox-android:4.0.2"
5656
}
5757
}

flutter_libs/ios/objectbox_flutter_libs.podspec

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Pod::Spec.new do |s|
1818
s.source_files = 'Classes/**/*'
1919

2020
s.dependency 'Flutter'
21-
s.dependency 'ObjectBox', '2.0.0'
21+
s.dependency 'ObjectBox', '4.0.0'
2222

2323
# Flutter.framework does not contain a i386 slice.
2424
s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' }

flutter_libs/linux/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ target_link_libraries(${PLUGIN_NAME} PRIVATE PkgConfig::GTK)
4444
# ----------------------------------------------------------------------
4545
# Download and add objectbox-c prebuilt library.
4646

47-
set(OBJECTBOX_VERSION 4.0.0)
47+
set(OBJECTBOX_VERSION 4.0.1)
4848

4949
set(OBJECTBOX_ARCH ${CMAKE_SYSTEM_PROCESSOR})
5050
if (${OBJECTBOX_ARCH} MATCHES "x86_64")

flutter_libs/macos/objectbox_flutter_libs.podspec

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Pod::Spec.new do |s|
1818
s.source_files = 'Classes/**/*'
1919

2020
s.dependency 'FlutterMacOS'
21-
s.dependency 'ObjectBox', '2.0.0'
21+
s.dependency 'ObjectBox', '4.0.0'
2222

2323
s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }
2424
s.swift_version = '5.3'

flutter_libs/windows/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ set(objectbox_flutter_libs_bundled_libraries
5050
# ----------------------------------------------------------------------
5151
# Download and add objectbox-c prebuilt library.
5252

53-
set(OBJECTBOX_VERSION 4.0.0)
53+
set(OBJECTBOX_VERSION 4.0.1)
5454

5555
set(OBJECTBOX_ARCH ${CMAKE_SYSTEM_PROCESSOR})
5656
if (${OBJECTBOX_ARCH} MATCHES "AMD64")

install.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -eu
55
# It's important that the generated dart bindings and the c-api library version match. Dart won't error on C function
66
# signature mismatch, leading to obscure memory bugs.
77
# For how to upgrade the version see dev-doc/updating-c-library.md
8-
cLibVersion=4.0.0
8+
cLibVersion=4.0.1
99
os=$(uname)
1010
cLibArgs="$*"
1111

objectbox/CHANGELOG.md

+8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
## latest
22

33
* Generator: replace cryptography library, allows to use newer versions of the transitive `js` dependency. [#638](https://github.com/objectbox/objectbox-dart/issues/638)
4+
* iOS: support `Query.findWithScores()` with big objects (> 4 KB), previously would throw a
5+
`StorageException: Do not use vector-based find on 32 bit systems with big objects`. [#676](https://github.com/objectbox/objectbox-dart/issues/676)
6+
* Flutter for Linux/Windows, Dart Native: update to [objectbox-c 0.4.1](https://github.com/objectbox/objectbox-c/releases/tag/v0.4.1).
7+
* Flutter for iOS/macOS: update to [objectbox-swift 4.0.0](https://github.com/objectbox/objectbox-swift/releases/tag/v4.0.0).
8+
Existing projects may have to run `pod repo update` and `pod update ObjectBox`.
9+
* Flutter for Android: update to [objectbox-android 4.0.2](https://github.com/objectbox/objectbox-java/releases/tag/V4.0.2).
10+
If you are [using Admin](https://docs.objectbox.io/data-browser#admin-for-android), make sure to
11+
update to `io.objectbox:objectbox-android-objectbrowser:4.0.2` in `android/app/build.gradle`.
412

513
## 4.0.2 (2024-08-14)
614

objectbox/example/flutter/objectbox_demo_relations/android/app/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -82,5 +82,5 @@ dependencies {
8282
// Add objectbox-android-objectbrowser only for debug builds.
8383
// Warning: when objectbox_flutter_libs updates check if version
8484
// needs update, e.g. check https://github.com/objectbox/objectbox-dart/releases.
85-
debugImplementation("io.objectbox:objectbox-android-objectbrowser:4.0.0")
85+
debugImplementation("io.objectbox:objectbox-android-objectbrowser:4.0.2")
8686
}

objectbox/lib/src/native/bindings/bindings.dart

+7-6
Original file line numberDiff line numberDiff line change
@@ -93,15 +93,16 @@ ObjectBoxC? _tryObjectBoxLibFile() {
9393
}
9494

9595
// Require the minimum C API version of all supported platform-specific
96-
// libraries:
97-
// objectbox-c: 4.0.0 (4.0.0-2024-05-14)
98-
// ObjectBox Swift 2.0.0: 4.0.0 (4.0.0-2024-05-14)
99-
// objectbox-android 4.0.0: 4.0.0 (4.0.0-2024-05-14)
96+
// libraries.
97+
// Library | C API version | Core version
98+
// objectbox-c | 4.0.1 | 4.0.1-2024-07-17
99+
// ObjectBox Swift 4.0.0 | 4.0.1 | 4.0.1-2024-07-17
100+
// objectbox-android 4.0.2 | 4.0.1 | 4.0.2-2024-08-19
100101
var _obxCminMajor = 4;
101102
var _obxCminMinor = 0;
102-
var _obxCminPatch = 0;
103+
var _obxCminPatch = 1;
103104
// Require minimum core version guaranteeing actual C API availability.
104-
var _obxCoreMinVersion = "4.0.0-2024-05-14";
105+
var _obxCoreMinVersion = "4.0.1-2024-07-17";
105106

106107
bool _isSupportedVersion(ObjectBoxC obxc) {
107108
if (!obxc.version_is_at_least(_obxCminMajor, _obxCminMinor, _obxCminPatch)) {

objectbox/lib/src/native/bindings/data_visitor.dart

+53-13
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@ import 'dart:ffi';
33
import 'bindings.dart';
44
import 'helpers.dart';
55

6-
/// Callback for reading data one-by-one, see [visit].
6+
/// Callback for reading query results one-by-one, see [visit].
77
typedef VisitCallback = bool Function(Pointer<Uint8> data, int size);
88

9+
/// Callback for reading query results one-by-one, see [visitWithScore].
10+
typedef VisitWithScoreCallback = bool Function(Pointer<OBX_bytes_score> data);
11+
912
/// Currently FFI's Pointer.fromFunction only allows to pass a static Dart
1013
/// callback function. When passing a closure it would throw at runtime:
1114
/// Error: fromFunction expects a static function as parameter.
@@ -18,31 +21,68 @@ typedef VisitCallback = bool Function(Pointer<Uint8> data, int size);
1821
/// and visits its results (e.g. query run in entity constructor or setter) and
1922
/// - Dart code within an isolate is executed synchronously:
2023
///
21-
/// Create a single static callback function [_callbackWrapper] that wraps
24+
/// Create a single static callback function [_visitCallbackWrapper] that wraps
2225
/// the actual Dart callback of the query currently visiting results.
23-
/// Keep callbacks on a [_callbackStack] to restore the callback of an outer
26+
/// Keep callbacks on a [_visitCallbackStack] to restore the callback of an outer
2427
/// query once a nested query is finished visiting results.
25-
List<VisitCallback> _callbackStack = [];
28+
List<VisitCallback> _visitCallbackStack = [];
29+
30+
/// Like [_visitCallbackStack], but for [VisitWithScoreCallback].
31+
List<VisitWithScoreCallback> _visitWithScoreCallbackStack = [];
32+
33+
bool _visitCallbackWrapper(Pointer<Uint8> dataPtr, int size, Pointer<Void> _) =>
34+
_visitCallbackStack.last(dataPtr, size);
2635

27-
bool _callbackWrapper(Pointer<Uint8> dataPtr, int size, Pointer<Void> _) =>
28-
_callbackStack.last(dataPtr, size);
36+
bool _visitWithScoreCallbackWrapper(
37+
Pointer<OBX_bytes_score> dataPtr, Pointer<Void> _) =>
38+
_visitWithScoreCallbackStack.last(dataPtr);
2939

30-
final Pointer<obx_data_visitor> _callbackWrapperPtr =
31-
Pointer.fromFunction(_callbackWrapper, false);
40+
final Pointer<obx_data_visitor> _visitCallbackWrapperPtr =
41+
Pointer.fromFunction(_visitCallbackWrapper, false);
3242

33-
/// Visits query results.
43+
final Pointer<obx_data_score_visitor> _visitWithScoreCallbackWrapperPtr =
44+
Pointer.fromFunction(_visitWithScoreCallbackWrapper, false);
45+
46+
/// Visits query results to read results one by one (in chunks).
47+
///
48+
/// This is useful to support large objects in 32-bit mode.
3449
///
35-
/// Pass a [callback] for reading data one-by-one:
50+
/// Pass a [callback] for reading data one by one:
3651
/// - [data] is the read data buffer.
3752
/// - [size] specifies the length of the read data.
3853
/// - Return true to keep going, false to cancel.
54+
///
55+
/// Use [ObjectVisitorError] to get an error out of the callback.
3956
@pragma('vm:prefer-inline')
4057
void visit(Pointer<OBX_query> queryPtr, VisitCallback callback) {
4158
// Keep callback in case another query is created and visits results
4259
// within the callback.
43-
_callbackStack.add(callback);
44-
final code = C.query_visit(queryPtr, _callbackWrapperPtr, nullptr);
45-
_callbackStack.removeLast();
60+
_visitCallbackStack.add(callback);
61+
final code = C.query_visit(queryPtr, _visitCallbackWrapperPtr, nullptr);
62+
_visitCallbackStack.removeLast();
63+
// Clean callback from stack before potentially throwing.
64+
checkObx(code);
65+
}
66+
67+
/// Visits query with score results to read results one by one (in chunks).
68+
///
69+
/// This is useful to support large objects in 32-bit mode.
70+
///
71+
/// Pass a [callback] for reading data one by one.
72+
/// - [data] is a [OBX_bytes_score] that iself contains data of the object and
73+
/// the length of the data.
74+
/// - Return true to keep going, false to cancel.
75+
///
76+
/// Use [ObjectVisitorError] to get an error out of the callback.
77+
@pragma('vm:prefer-inline')
78+
void visitWithScore(
79+
Pointer<OBX_query> queryPtr, VisitWithScoreCallback callback) {
80+
// Keep callback in case another query is created and visits results
81+
// within the callback.
82+
_visitWithScoreCallbackStack.add(callback);
83+
final code = C.query_visit_with_score(
84+
queryPtr, _visitWithScoreCallbackWrapperPtr, nullptr);
85+
_visitWithScoreCallbackStack.removeLast();
4686
// Clean callback from stack before potentially throwing.
4787
checkObx(code);
4888
}

objectbox/lib/src/native/bindings/objectbox-sync.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
#include "objectbox.h"
3535

3636
#if defined(static_assert) || defined(__cplusplus)
37-
static_assert(OBX_VERSION_MAJOR == 4 && OBX_VERSION_MINOR == 0 && OBX_VERSION_PATCH == 0, // NOLINT
37+
static_assert(OBX_VERSION_MAJOR == 4 && OBX_VERSION_MINOR == 0 && OBX_VERSION_PATCH == 1, // NOLINT
3838
"Versions of objectbox.h and objectbox-sync.h files do not match, please update");
3939
#endif
4040

objectbox/lib/src/native/bindings/objectbox.h

+67-5
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ extern "C" {
5353
/// obx_version() or obx_version_is_at_least().
5454
#define OBX_VERSION_MAJOR 4
5555
#define OBX_VERSION_MINOR 0
56-
#define OBX_VERSION_PATCH 0 // values >= 100 are reserved for dev releases leading to the next minor/major increase
56+
#define OBX_VERSION_PATCH 1 // values >= 100 are reserved for dev releases leading to the next minor/major increase
5757

5858
//----------------------------------------------
5959
// Common types
@@ -79,13 +79,19 @@ typedef struct OBX_id_score {
7979
/// Error/success code returned by an obx_* function; see defines OBX_SUCCESS, OBX_NOT_FOUND, and OBX_ERROR_*
8080
typedef int obx_err;
8181

82-
/// The callback for reading data one-by-one
82+
/// The callback for reading data (i.e. object bytes) one-by-one.
8383
/// @param data is the read data buffer
8484
/// @param size specifies the length of the read data
8585
/// @param user_data is a pass-through argument passed to the called API
86-
/// @return true to keep going, false to cancel.
86+
/// @return The visitor returns true to keep going or false to cancel.
8787
typedef bool obx_data_visitor(const uint8_t* data, size_t size, void* user_data);
8888

89+
/// The callback for reading data (i.e. object bytes) with a search score one-by-one.
90+
/// @param data contains the current data with score element
91+
/// @param user_data is a pass-through argument passed to the called API
92+
/// @return The visitor returns true to keep going or false to cancel.
93+
typedef bool obx_data_score_visitor(const struct OBX_bytes_score* data, void* user_data);
94+
8995
//----------------------------------------------
9096
// Runtime library information
9197
//
@@ -805,6 +811,15 @@ typedef enum {
805811
OBXBackupFlags_ExcludeSalt = 0x2,
806812
} OBXBackupFlags;
807813

814+
/// WAL flags control how the store handles WAL files.
815+
typedef enum {
816+
/// Enable Wal
817+
OBXWalFlags_EnableWal = 0x1,
818+
819+
/// Does not wait for the disk to acknowledge; faster but not ACID compliant (not generally recommended).
820+
OBXWalFlags_NoSyncFile = 0x2,
821+
} OBXWalFlags;
822+
808823
/// This bytes struct is an input/output wrapper used for a single data object (represented as FlatBuffers).
809824
typedef struct OBX_bytes {
810825
const uint8_t* data;
@@ -1087,6 +1102,23 @@ typedef enum {
10871102
/// e.g., to overwrite all existing data in the database.
10881103
OBX_C_API void obx_opt_backup_restore(OBX_store_options* opt, const char* backup_file, uint32_t flags);
10891104

1105+
/// Enables Write-ahead logging (WAL) if OBXWalFlags_EnableWal is given.
1106+
/// For now this is only supported for in-memory DBs.
1107+
/// @param flags OBXWalFlags_EnableWal with optional other flags (bitwise OR).
1108+
OBX_C_API void obx_opt_wal(OBX_store_options* opt, uint32_t flags);
1109+
1110+
/// The WAL file gets consolidated when it reached this size limit when opening the database.
1111+
/// This setting is meant for applications that prefer to consolidate on startup,
1112+
/// which may avoid consolidations on commits while the application is running.
1113+
/// The default is 4096 (4 MB).
1114+
OBX_C_API void obx_opt_wal_max_file_size_on_open_in_kb(OBX_store_options* opt, uint64_t size_in_kb);
1115+
1116+
/// The WAL file gets consolidated when it reaches this size limit after a commit.
1117+
/// As consolidation takes some time, it is a trade-off between accumulating enough data
1118+
/// and the time the consolidation takes (longer with more data).
1119+
/// The default is 16384 (16 MB).
1120+
OBX_C_API void obx_opt_wal_max_file_size_in_kb(OBX_store_options* opt, uint64_t size_in_kb);
1121+
10901122
/// Gets the option for "directory"; this is either the default, or, the value set by obx_opt_directory().
10911123
/// The returned value must not be modified and is only valid for the lifetime of the options or until the value is
10921124
/// changed.
@@ -1235,7 +1267,10 @@ typedef enum {
12351267
OBXStoreTypeId_LMDB = 1,
12361268

12371269
/// Store type ID for in-memory database (non-persistent)
1238-
OBXStoreTypeId_InMemory = 2
1270+
OBXStoreTypeId_InMemory = 2,
1271+
1272+
/// Store type ID for in-memory WAL-enabled (persistent)
1273+
OBXStoreTypeId_InMemoryWal = 3
12391274

12401275
} OBXStoreTypeId;
12411276

@@ -2053,10 +2088,14 @@ OBX_C_API obx_err obx_query_find_first(OBX_query* query, const uint8_t** data, s
20532088
/// operation (e.g. put/remove) was executed. Accessing data after this is undefined behavior.
20542089
OBX_C_API obx_err obx_query_find_unique(OBX_query* query, const uint8_t** data, size_t* size);
20552090

2056-
/// Walk over matching objects using the given data visitor.
2091+
/// Walk over matching objects one-by-one using the given data visitor (a callback function).
20572092
/// Note: if no order conditions is present, the order is arbitrary (sometimes ordered by ID, but never guaranteed to).
20582093
OBX_C_API obx_err obx_query_visit(OBX_query* query, obx_data_visitor* visitor, void* user_data);
20592094

2095+
/// Walk over matching objects with their query score one-by-one using the given data visitor (a callback function).
2096+
/// Note: the elements are ordered by the score (ascending).
2097+
OBX_C_API obx_err obx_query_visit_with_score(OBX_query* query, obx_data_score_visitor* visitor, void* user_data);
2098+
20602099
/// Return the IDs of all matching objects.
20612100
/// Note: if no order conditions is present, the order is arbitrary (sometimes ordered by ID, but never guaranteed to).
20622101
OBX_C_API OBX_id_array* obx_query_find_ids(OBX_query* query);
@@ -2492,6 +2531,29 @@ OBX_C_API obx_id obx_tree_leaves_info_id(OBX_tree_leaves_info* leaves_info, size
24922531
/// Frees a leaves info reference.
24932532
OBX_C_API void obx_tree_leaves_info_free(OBX_tree_leaves_info* leaves_info);
24942533

2534+
/// Callback for obx_tree_async_get_raw().
2535+
/// \note If the given status is an error, you can use functions like obx_last_error_message() to gather more info
2536+
/// during this callback (error state is thread bound and the callback uses an internal thread).
2537+
/// @param status The result status of the async operation
2538+
/// @param id If the operation was successful, the ID of the leaf, which was get (otherwise zero).
2539+
/// @param path The leafs path as string.
2540+
/// @param leaf_data The leafs data flatbuffer pointer.
2541+
/// @param leaf_data_size The leafs data flatbuffer size.
2542+
/// @param leaf_metadata The leafs metadata flatbuffer pointer.
2543+
/// @param leaf_metadata_size The leafs meatdata flatbuffer size.
2544+
/// @param user_data The data initially passed to the async function call is passed back.
2545+
typedef void obx_tree_async_get_callback(obx_err status, obx_id id, const char* path, const uint8_t* leaf_data,
2546+
size_t leaf_data_size, const uint8_t* leaf_metadata, size_t leaf_metadata_size,
2547+
void* user_data);
2548+
2549+
/// Like obx_tree_cursor_get_raw(), but asynchronous.
2550+
/// @param with_metadata Flag if the callback also wants to receive the metadata (also as raw FlatBuffers).
2551+
/// @param callback Optional (may be null) function that is called with results once the async operation completes.
2552+
/// @param callback_user_data Any value you can supply, which is passed on to the callback (e.g. to identify user
2553+
/// specific context).
2554+
OBX_C_API obx_err obx_tree_async_get_raw(OBX_tree* tree, const char* path, bool with_metadata,
2555+
obx_tree_async_get_callback* callback, void* callback_user_data);
2556+
24952557
/// Callback for obx_tree_async_put_raw().
24962558
/// \note If the given status is an error, you can use functions like obx_last_error_message() to gather more info
24972559
/// during this callback (error state is thread bound and the callback uses an internal thread).

0 commit comments

Comments
 (0)