diff --git a/croaring-sys/CRoaring/bindgen_bundled_version.rs b/croaring-sys/CRoaring/bindgen_bundled_version.rs index ac27fe0..846f5e7 100644 --- a/croaring-sys/CRoaring/bindgen_bundled_version.rs +++ b/croaring-sys/CRoaring/bindgen_bundled_version.rs @@ -1,9 +1,9 @@ -/* automatically generated by rust-bindgen 0.68.1 */ +/* automatically generated by rust-bindgen 0.69.4 */ -pub const ROARING_VERSION: &[u8; 6] = b"2.0.2\0"; -pub const ROARING_VERSION_MAJOR: _bindgen_ty_1 = 2; +pub const ROARING_VERSION: &[u8; 6] = b"3.0.1\0"; +pub const ROARING_VERSION_MAJOR: _bindgen_ty_1 = 3; pub const ROARING_VERSION_MINOR: _bindgen_ty_1 = 0; -pub const ROARING_VERSION_REVISION: _bindgen_ty_1 = 2; +pub const ROARING_VERSION_REVISION: _bindgen_ty_1 = 1; pub type _bindgen_ty_1 = ::std::os::raw::c_uint; #[doc = " Roaring arrays are array-based key-value pairs having containers as values\n and 16-bit integer keys. A roaring bitmap might be implemented as such."] #[repr(C)] @@ -45,6 +45,23 @@ pub struct roaring_statistics_s { } #[doc = " (For advanced users.)\n The roaring_statistics_t can be used to collect detailed statistics about\n the composition of a roaring bitmap."] pub type roaring_statistics_t = roaring_statistics_s; +#[doc = " Roaring-internal type used to iterate within a roaring container."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct roaring_container_iterator_s { + pub index: i32, +} +#[doc = " Roaring-internal type used to iterate within a roaring container."] +pub type roaring_container_iterator_t = roaring_container_iterator_s; +extern "C" { + #[doc = " result might be undefined when input_num is zero"] + pub fn roaring_trailing_zeroes(input_num: ::std::os::raw::c_ulonglong) + -> ::std::os::raw::c_int; +} +extern "C" { + #[doc = " result might be undefined when input_num is zero"] + pub fn roaring_leading_zeroes(input_num: ::std::os::raw::c_ulonglong) -> ::std::os::raw::c_int; +} #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct bitset_s { @@ -217,7 +234,7 @@ extern "C" { pub fn roaring_bitmap_printf_describe(r: *const roaring_bitmap_t); } extern "C" { - #[doc = " Creates a new bitmap from a list of uint32_t integers"] + #[doc = " Creates a new bitmap from a list of uint32_t integers\n\n This function is deprecated, use `roaring_bitmap_from` instead, which\n doesn't require the number of elements to be passed in.\n\n @see roaring_bitmap_from"] pub fn roaring_bitmap_of(n: usize, ...) -> *mut roaring_bitmap_t; } extern "C" { @@ -225,7 +242,7 @@ extern "C" { pub fn roaring_bitmap_copy(r: *const roaring_bitmap_t) -> *mut roaring_bitmap_t; } extern "C" { - #[doc = " Copies a bitmap from src to dest. It is assumed that the pointer dest\n is to an already allocated bitmap. The content of the dest bitmap is\n freed/deleted.\n\n It might be preferable and simpler to call roaring_bitmap_copy except\n that roaring_bitmap_overwrite can save on memory allocations.\n\n Returns true if successful, or false if there was an error. On failure,\n the dest bitmap is left in a valid, empty state (even if it was not empty before)."] + #[doc = " Copies a bitmap from src to dest. It is assumed that the pointer dest\n is to an already allocated bitmap. The content of the dest bitmap is\n freed/deleted.\n\n It might be preferable and simpler to call roaring_bitmap_copy except\n that roaring_bitmap_overwrite can save on memory allocations.\n\n Returns true if successful, or false if there was an error. On failure,\n the dest bitmap is left in a valid, empty state (even if it was not empty\n before)."] pub fn roaring_bitmap_overwrite( dest: *mut roaring_bitmap_t, src: *const roaring_bitmap_t, @@ -257,7 +274,7 @@ extern "C" { ) -> bool; } extern "C" { - #[doc = " Check whether a bitmap and a closed range intersect."] + #[doc = " Check whether a bitmap and an open range intersect."] pub fn roaring_bitmap_intersect_with_range(bm: *const roaring_bitmap_t, x: u64, y: u64) -> bool; } @@ -459,7 +476,7 @@ extern "C" { pub fn roaring_bitmap_to_bitset(r: *const roaring_bitmap_t, bitset: *mut bitset_t) -> bool; } extern "C" { - #[doc = " Convert the bitmap to a sorted array from `offset` by `limit`, output in `ans`.\n\n Caller is responsible to ensure that there is enough memory allocated, e.g.\n\n ans = malloc(roaring_bitmap_get_cardinality(limit) * sizeof(uint32_t));\n\n Return false in case of failure (e.g., insufficient memory)"] + #[doc = " Convert the bitmap to a sorted array from `offset` by `limit`, output in\n `ans`.\n\n Caller is responsible to ensure that there is enough memory allocated, e.g.\n\n ans = malloc(roaring_bitmap_get_cardinality(limit) * sizeof(uint32_t));\n\n Return false in case of failure (e.g., insufficient memory)"] pub fn roaring_bitmap_range_uint32_array( r: *const roaring_bitmap_t, offset: usize, @@ -480,18 +497,18 @@ extern "C" { pub fn roaring_bitmap_shrink_to_fit(r: *mut roaring_bitmap_t) -> usize; } extern "C" { - #[doc = " Write the bitmap to an output pointer, this output buffer should refer to\n at least `roaring_bitmap_size_in_bytes(r)` allocated bytes.\n\n See `roaring_bitmap_portable_serialize()` if you want a format that's\n compatible with Java and Go implementations. This format can sometimes be\n more space efficient than the portable form, e.g. when the data is sparse.\n\n Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Write the bitmap to an output pointer, this output buffer should refer to\n at least `roaring_bitmap_size_in_bytes(r)` allocated bytes.\n\n See `roaring_bitmap_portable_serialize()` if you want a format that's\n compatible with Java and Go implementations. This format can sometimes be\n more space efficient than the portable form, e.g. when the data is sparse.\n\n Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_serialize( r: *const roaring_bitmap_t, buf: *mut ::std::os::raw::c_char, ) -> usize; } extern "C" { - #[doc = " Use with `roaring_bitmap_serialize()`.\n\n (See `roaring_bitmap_portable_deserialize()` if you want a format that's\n compatible with Java and Go implementations).\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Use with `roaring_bitmap_serialize()`.\n\n (See `roaring_bitmap_portable_deserialize()` if you want a format that's\n compatible with Java and Go implementations).\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_deserialize(buf: *const ::std::os::raw::c_void) -> *mut roaring_bitmap_t; } extern "C" { - #[doc = " Use with `roaring_bitmap_serialize()`.\n\n (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's\n compatible with Java and Go implementations).\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems.\n\n The difference with `roaring_bitmap_deserialize()` is that this function checks that the input buffer\n is a valid bitmap. If the buffer is too small, NULL is returned."] + #[doc = " Use with `roaring_bitmap_serialize()`.\n\n (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's\n compatible with Java and Go implementations).\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems.\n\n The difference with `roaring_bitmap_deserialize()` is that this function\n checks that the input buffer is a valid bitmap. If the buffer is too small,\n NULL is returned."] pub fn roaring_bitmap_deserialize_safe( buf: *const ::std::os::raw::c_void, maxbytes: usize, @@ -502,20 +519,20 @@ extern "C" { pub fn roaring_bitmap_size_in_bytes(r: *const roaring_bitmap_t) -> usize; } extern "C" { - #[doc = " Read bitmap from a serialized buffer.\n In case of failure, NULL is returned.\n\n This function is unsafe in the sense that if there is no valid serialized\n bitmap at the pointer, then many bytes could be read, possibly causing a\n buffer overflow. See also roaring_bitmap_portable_deserialize_safe().\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Read bitmap from a serialized buffer.\n In case of failure, NULL is returned.\n\n This function is unsafe in the sense that if there is no valid serialized\n bitmap at the pointer, then many bytes could be read, possibly causing a\n buffer overflow. See also roaring_bitmap_portable_deserialize_safe().\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_portable_deserialize( buf: *const ::std::os::raw::c_char, ) -> *mut roaring_bitmap_t; } extern "C" { - #[doc = " Read bitmap from a serialized buffer safely (reading up to maxbytes).\n In case of failure, NULL is returned.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n The function itself is safe in the sense that it will not cause buffer overflows.\n However, for correct operations, it is assumed that the bitmap read was once\n serialized from a valid bitmap (i.e., it follows the format specification).\n If you provided an incorrect input (garbage), then the bitmap read may not be in\n a valid state and following operations may not lead to sensible results.\n In particular, the serialized array containers need to be in sorted order, and the\n run containers should be in sorted non-overlapping order. This is is guaranteed to\n happen when serializing an existing bitmap, but not for random inputs.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Read bitmap from a serialized buffer safely (reading up to maxbytes).\n In case of failure, NULL is returned.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n The function itself is safe in the sense that it will not cause buffer\n overflows. However, for correct operations, it is assumed that the bitmap\n read was once serialized from a valid bitmap (i.e., it follows the format\n specification). If you provided an incorrect input (garbage), then the bitmap\n read may not be in a valid state and following operations may not lead to\n sensible results. In particular, the serialized array containers need to be\n in sorted order, and the run containers should be in sorted non-overlapping\n order. This is is guaranteed to happen when serializing an existing bitmap,\n but not for random inputs.\n\n You may use roaring_bitmap_internal_validate to check the validity of the\n bitmap prior to using it. You may also use other strategies to check for\n corrupted inputs (e.g., checksums).\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_portable_deserialize_safe( buf: *const ::std::os::raw::c_char, maxbytes: usize, ) -> *mut roaring_bitmap_t; } extern "C" { - #[doc = " Read bitmap from a serialized buffer.\n In case of failure, NULL is returned.\n\n Bitmap returned by this function can be used in all readonly contexts.\n Bitmap must be freed as usual, by calling roaring_bitmap_free().\n Underlying buffer must not be freed or modified while it backs any bitmaps.\n\n The function is unsafe in the following ways:\n 1) It may execute unaligned memory accesses.\n 2) A buffer overflow may occur if buf does not point to a valid serialized\n bitmap.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Read bitmap from a serialized buffer.\n In case of failure, NULL is returned.\n\n Bitmap returned by this function can be used in all readonly contexts.\n Bitmap must be freed as usual, by calling roaring_bitmap_free().\n Underlying buffer must not be freed or modified while it backs any bitmaps.\n\n The function is unsafe in the following ways:\n 1) It may execute unaligned memory accesses.\n 2) A buffer overflow may occur if buf does not point to a valid serialized\n bitmap.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_portable_deserialize_frozen( buf: *const ::std::os::raw::c_char, ) -> *mut roaring_bitmap_t; @@ -532,7 +549,7 @@ extern "C" { pub fn roaring_bitmap_portable_size_in_bytes(r: *const roaring_bitmap_t) -> usize; } extern "C" { - #[doc = " Write a bitmap to a char buffer. The output buffer should refer to at least\n `roaring_bitmap_portable_size_in_bytes(r)` bytes of allocated memory.\n\n Returns how many bytes were written which should match\n `roaring_bitmap_portable_size_in_bytes(r)`.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Write a bitmap to a char buffer. The output buffer should refer to at least\n `roaring_bitmap_portable_size_in_bytes(r)` bytes of allocated memory.\n\n Returns how many bytes were written which should match\n `roaring_bitmap_portable_size_in_bytes(r)`.\n\n This is meant to be compatible with the Java and Go versions:\n https://github.com/RoaringBitmap/RoaringFormatSpec\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_portable_serialize( r: *const roaring_bitmap_t, buf: *mut ::std::os::raw::c_char, @@ -543,14 +560,14 @@ extern "C" { pub fn roaring_bitmap_frozen_size_in_bytes(r: *const roaring_bitmap_t) -> usize; } extern "C" { - #[doc = " Serializes bitmap using frozen format.\n Buffer size must be at least roaring_bitmap_frozen_size_in_bytes().\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Serializes bitmap using frozen format.\n Buffer size must be at least roaring_bitmap_frozen_size_in_bytes().\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_frozen_serialize( r: *const roaring_bitmap_t, buf: *mut ::std::os::raw::c_char, ); } extern "C" { - #[doc = " Creates constant bitmap that is a view of a given buffer.\n Buffer data should have been written by `roaring_bitmap_frozen_serialize()`\n Its beginning must also be aligned by 32 bytes.\n Length must be equal exactly to `roaring_bitmap_frozen_size_in_bytes()`.\n In case of failure, NULL is returned.\n\n Bitmap returned by this function can be used in all readonly contexts.\n Bitmap must be freed as usual, by calling roaring_bitmap_free().\n Underlying buffer must not be freed or modified while it backs any bitmaps.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x),\n the data format is going to be big-endian and not compatible with little-endian systems."] + #[doc = " Creates constant bitmap that is a view of a given buffer.\n Buffer data should have been written by `roaring_bitmap_frozen_serialize()`\n Its beginning must also be aligned by 32 bytes.\n Length must be equal exactly to `roaring_bitmap_frozen_size_in_bytes()`.\n In case of failure, NULL is returned.\n\n Bitmap returned by this function can be used in all readonly contexts.\n Bitmap must be freed as usual, by calling roaring_bitmap_free().\n Underlying buffer must not be freed or modified while it backs any bitmaps.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] pub fn roaring_bitmap_frozen_view( buf: *const ::std::os::raw::c_char, length: usize, @@ -641,6 +658,15 @@ extern "C" { #[doc = " roaring_bitmap_rank returns the number of integers that are smaller or equal\n to x. Thus if x is the first element, this function will return 1. If\n x is smaller than the smallest element, this function will return 0.\n\n The indexing convention differs between roaring_bitmap_select and\n roaring_bitmap_rank: roaring_bitmap_select refers to the smallest value\n as having index 0, whereas roaring_bitmap_rank returns 1 when ranking\n the smallest value."] pub fn roaring_bitmap_rank(r: *const roaring_bitmap_t, x: u32) -> u64; } +extern "C" { + #[doc = " roaring_bitmap_rank_many is an `Bulk` version of `roaring_bitmap_rank`\n it puts rank value of each element in `[begin .. end)` to `ans[]`\n\n the values in `[begin .. end)` must be sorted in Ascending order;\n Caller is responsible to ensure that there is enough memory allocated, e.g.\n\n ans = malloc((end-begin) * sizeof(uint64_t));"] + pub fn roaring_bitmap_rank_many( + r: *const roaring_bitmap_t, + begin: *const u32, + end: *const u32, + ans: *mut u64, + ); +} extern "C" { #[doc = " Returns the index of x in the given roaring bitmap.\n If the roaring bitmap doesn't contain x , this function will return -1.\n The difference with rank function is that this function will return -1 when x\n is not the element of roaring bitmap, but the rank function will return a\n non-negative number."] pub fn roaring_bitmap_get_index(r: *const roaring_bitmap_t, x: u32) -> i64; @@ -658,70 +684,69 @@ extern "C" { pub fn roaring_bitmap_statistics(r: *const roaring_bitmap_t, stat: *mut roaring_statistics_t); } extern "C" { - #[doc = " Perform internal consistency checks. Returns true if the bitmap is consistent.\n\n Note that some operations intentionally leave bitmaps in an inconsistent state temporarily,\n for example, `roaring_bitmap_lazy_*` functions, until `roaring_bitmap_repair_after_lazy` is called.\n\n If reason is non-null, it will be set to a string describing the first inconsistency found if any."] + #[doc = " Perform internal consistency checks. Returns true if the bitmap is\n consistent. It may be useful to call this after deserializing bitmaps from\n untrusted sources. If roaring_bitmap_internal_validate returns true, then the\n bitmap should be consistent and can be trusted not to cause crashes or memory\n corruption.\n\n Note that some operations intentionally leave bitmaps in an inconsistent\n state temporarily, for example, `roaring_bitmap_lazy_*` functions, until\n `roaring_bitmap_repair_after_lazy` is called.\n\n If reason is non-null, it will be set to a string describing the first\n inconsistency found if any."] pub fn roaring_bitmap_internal_validate( r: *const roaring_bitmap_t, reason: *mut *const ::std::os::raw::c_char, ) -> bool; } -#[doc = " What follows is code use to iterate through values in a roaring bitmap\n\nroaring_bitmap_t *r =...\nroaring_uint32_iterator_t i;\nroaring_create_iterator(r, &i);\nwhile(i.has_value) {\nprintf(\"value = %d\\n\", i.current_value);\nroaring_advance_uint32_iterator(&i);\n}\n\nObviously, if you modify the underlying bitmap, the iterator\nbecomes invalid. So don't."] +#[doc = " A struct used to keep iterator state. Users should only access\n `current_value` and `has_value`, the rest of the type should be treated as\n opaque."] #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct roaring_uint32_iterator_s { pub parent: *const roaring_bitmap_t, - pub container_index: i32, - pub in_container_index: i32, - pub run_index: i32, - pub current_value: u32, - pub has_value: bool, pub container: *const ::std::os::raw::c_void, pub typecode: u8, + pub container_index: i32, pub highbits: u32, + pub container_it: roaring_container_iterator_t, + pub current_value: u32, + pub has_value: bool, } -#[doc = " What follows is code use to iterate through values in a roaring bitmap\n\nroaring_bitmap_t *r =...\nroaring_uint32_iterator_t i;\nroaring_create_iterator(r, &i);\nwhile(i.has_value) {\nprintf(\"value = %d\\n\", i.current_value);\nroaring_advance_uint32_iterator(&i);\n}\n\nObviously, if you modify the underlying bitmap, the iterator\nbecomes invalid. So don't."] +#[doc = " A struct used to keep iterator state. Users should only access\n `current_value` and `has_value`, the rest of the type should be treated as\n opaque."] pub type roaring_uint32_iterator_t = roaring_uint32_iterator_s; extern "C" { - #[doc = " Initialize an iterator object that can be used to iterate through the\n values. If there is a value, then this iterator points to the first value\n and `it->has_value` is true. The value is in `it->current_value`."] - pub fn roaring_init_iterator(r: *const roaring_bitmap_t, newit: *mut roaring_uint32_iterator_t); + #[doc = " Initialize an iterator object that can be used to iterate through the values.\n If there is a value, then this iterator points to the first value and\n `it->has_value` is true. The value is in `it->current_value`."] + pub fn roaring_iterator_init(r: *const roaring_bitmap_t, newit: *mut roaring_uint32_iterator_t); } extern "C" { - #[doc = " Initialize an iterator object that can be used to iterate through the\n values. If there is a value, then this iterator points to the last value\n and `it->has_value` is true. The value is in `it->current_value`."] - pub fn roaring_init_iterator_last( + #[doc = " Initialize an iterator object that can be used to iterate through the values.\n If there is a value, then this iterator points to the last value and\n `it->has_value` is true. The value is in `it->current_value`."] + pub fn roaring_iterator_init_last( r: *const roaring_bitmap_t, newit: *mut roaring_uint32_iterator_t, ); } extern "C" { - #[doc = " Create an iterator object that can be used to iterate through the values.\n Caller is responsible for calling `roaring_free_iterator()`.\n\n The iterator is initialized (this function calls `roaring_init_iterator()`)\n If there is a value, then this iterator points to the first value and\n `it->has_value` is true. The value is in `it->current_value`."] - pub fn roaring_create_iterator(r: *const roaring_bitmap_t) -> *mut roaring_uint32_iterator_t; + #[doc = " Create an iterator object that can be used to iterate through the values.\n Caller is responsible for calling `roaring_free_iterator()`.\n\n The iterator is initialized (this function calls `roaring_iterator_init()`)\n If there is a value, then this iterator points to the first value and\n `it->has_value` is true. The value is in `it->current_value`."] + pub fn roaring_iterator_create(r: *const roaring_bitmap_t) -> *mut roaring_uint32_iterator_t; } extern "C" { - #[doc = " Advance the iterator. If there is a new value, then `it->has_value` is true.\n The new value is in `it->current_value`. Values are traversed in increasing\n orders. For convenience, returns `it->has_value`."] - pub fn roaring_advance_uint32_iterator(it: *mut roaring_uint32_iterator_t) -> bool; + #[doc = " Advance the iterator. If there is a new value, then `it->has_value` is true.\n The new value is in `it->current_value`. Values are traversed in increasing\n orders. For convenience, returns `it->has_value`.\n\n Once `it->has_value` is false, `roaring_uint32_iterator_advance` should not\n be called on the iterator again. Calling `roaring_uint32_iterator_previous`\n is allowed."] + pub fn roaring_uint32_iterator_advance(it: *mut roaring_uint32_iterator_t) -> bool; } extern "C" { - #[doc = " Decrement the iterator. If there's a new value, then `it->has_value` is true.\n The new value is in `it->current_value`. Values are traversed in decreasing\n order. For convenience, returns `it->has_value`."] - pub fn roaring_previous_uint32_iterator(it: *mut roaring_uint32_iterator_t) -> bool; + #[doc = " Decrement the iterator. If there's a new value, then `it->has_value` is true.\n The new value is in `it->current_value`. Values are traversed in decreasing\n order. For convenience, returns `it->has_value`.\n\n Once `it->has_value` is false, `roaring_uint32_iterator_previous` should not\n be called on the iterator again. Calling `roaring_uint32_iterator_advance` is\n allowed."] + pub fn roaring_uint32_iterator_previous(it: *mut roaring_uint32_iterator_t) -> bool; } extern "C" { #[doc = " Move the iterator to the first value >= `val`. If there is a such a value,\n then `it->has_value` is true. The new value is in `it->current_value`.\n For convenience, returns `it->has_value`."] - pub fn roaring_move_uint32_iterator_equalorlarger( + pub fn roaring_uint32_iterator_move_equalorlarger( it: *mut roaring_uint32_iterator_t, val: u32, ) -> bool; } extern "C" { #[doc = " Creates a copy of an iterator.\n Caller must free it."] - pub fn roaring_copy_uint32_iterator( + pub fn roaring_uint32_iterator_copy( it: *const roaring_uint32_iterator_t, ) -> *mut roaring_uint32_iterator_t; } extern "C" { - #[doc = " Free memory following `roaring_create_iterator()`"] - pub fn roaring_free_uint32_iterator(it: *mut roaring_uint32_iterator_t); + #[doc = " Free memory following `roaring_iterator_create()`"] + pub fn roaring_uint32_iterator_free(it: *mut roaring_uint32_iterator_t); } extern "C" { - pub fn roaring_read_uint32_iterator( + pub fn roaring_uint32_iterator_read( it: *mut roaring_uint32_iterator_t, buf: *mut u32, count: u32, @@ -780,3 +805,417 @@ extern "C" { extern "C" { pub fn roaring_aligned_free(arg1: *mut ::std::os::raw::c_void); } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct roaring64_bitmap_s { + _unused: [u8; 0], +} +pub type roaring64_bitmap_t = roaring64_bitmap_s; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct roaring64_leaf_s { + _unused: [u8; 0], +} +pub type roaring64_leaf_t = roaring64_leaf_s; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct roaring64_iterator_s { + _unused: [u8; 0], +} +pub type roaring64_iterator_t = roaring64_iterator_s; +#[doc = " A bit of context usable with `roaring64_bitmap_*_bulk()` functions.\n\n Should be initialized with `{0}` (or `memset()` to all zeros).\n Callers should treat it as an opaque type.\n\n A context may only be used with a single bitmap (unless re-initialized to\n zero), and any modification to a bitmap (other than modifications performed\n with `_bulk()` functions with the context passed) will invalidate any\n contexts associated with that bitmap."] +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct roaring64_bulk_context_s { + pub high_bytes: [u8; 6usize], + pub leaf: *mut roaring64_leaf_t, +} +#[doc = " A bit of context usable with `roaring64_bitmap_*_bulk()` functions.\n\n Should be initialized with `{0}` (or `memset()` to all zeros).\n Callers should treat it as an opaque type.\n\n A context may only be used with a single bitmap (unless re-initialized to\n zero), and any modification to a bitmap (other than modifications performed\n with `_bulk()` functions with the context passed) will invalidate any\n contexts associated with that bitmap."] +pub type roaring64_bulk_context_t = roaring64_bulk_context_s; +extern "C" { + #[doc = " Dynamically allocates a new bitmap (initially empty).\n Client is responsible for calling `roaring64_bitmap_free()`."] + pub fn roaring64_bitmap_create() -> *mut roaring64_bitmap_t; +} +extern "C" { + pub fn roaring64_bitmap_free(r: *mut roaring64_bitmap_t); +} +extern "C" { + #[doc = " Returns a copy of a bitmap."] + pub fn roaring64_bitmap_copy(r: *const roaring64_bitmap_t) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Creates a new bitmap of a pointer to N 64-bit integers."] + pub fn roaring64_bitmap_of_ptr(n_args: usize, vals: *const u64) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Create a new bitmap containing all the values in [min, max) that are at a\n distance k*step from min."] + pub fn roaring64_bitmap_from_range(min: u64, max: u64, step: u64) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Adds the provided value to the bitmap."] + pub fn roaring64_bitmap_add(r: *mut roaring64_bitmap_t, val: u64); +} +extern "C" { + #[doc = " Adds the provided value to the bitmap.\n Returns true if a new value was added, false if the value already existed."] + pub fn roaring64_bitmap_add_checked(r: *mut roaring64_bitmap_t, val: u64) -> bool; +} +extern "C" { + #[doc = " Add an item, using context from a previous insert for faster insertion.\n\n `context` will be used to store information between calls to make bulk\n operations faster. `*context` should be zero-initialized before the first\n call to this function.\n\n Modifying the bitmap in any way (other than `-bulk` suffixed functions)\n will invalidate the stored context, calling this function with a non-zero\n context after doing any modification invokes undefined behavior.\n\n In order to exploit this optimization, the caller should call this function\n with values with the same high 48 bits of the value consecutively."] + pub fn roaring64_bitmap_add_bulk( + r: *mut roaring64_bitmap_t, + context: *mut roaring64_bulk_context_t, + val: u64, + ); +} +extern "C" { + #[doc = " Add `n_args` values from `vals`, faster than repeatedly calling\n `roaring64_bitmap_add()`\n\n In order to exploit this optimization, the caller should attempt to keep\n values with the same high 48 bits of the value as consecutive elements in\n `vals`."] + pub fn roaring64_bitmap_add_many(r: *mut roaring64_bitmap_t, n_args: usize, vals: *const u64); +} +extern "C" { + #[doc = " Add all values in range [min, max)."] + pub fn roaring64_bitmap_add_range(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " Add all values in range [min, max]."] + pub fn roaring64_bitmap_add_range_closed(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " Removes a value from the bitmap if present."] + pub fn roaring64_bitmap_remove(r: *mut roaring64_bitmap_t, val: u64); +} +extern "C" { + #[doc = " Removes a value from the bitmap if present, returns true if the value was\n removed and false if the value was not present."] + pub fn roaring64_bitmap_remove_checked(r: *mut roaring64_bitmap_t, val: u64) -> bool; +} +extern "C" { + #[doc = " Remove an item, using context from a previous insert for faster removal.\n\n `context` will be used to store information between calls to make bulk\n operations faster. `*context` should be zero-initialized before the first\n call to this function.\n\n Modifying the bitmap in any way (other than `-bulk` suffixed functions)\n will invalidate the stored context, calling this function with a non-zero\n context after doing any modification invokes undefined behavior.\n\n In order to exploit this optimization, the caller should call this function\n with values with the same high 48 bits of the value consecutively."] + pub fn roaring64_bitmap_remove_bulk( + r: *mut roaring64_bitmap_t, + context: *mut roaring64_bulk_context_t, + val: u64, + ); +} +extern "C" { + #[doc = " Remove `n_args` values from `vals`, faster than repeatedly calling\n `roaring64_bitmap_remove()`\n\n In order to exploit this optimization, the caller should attempt to keep\n values with the same high 48 bits of the value as consecutive elements in\n `vals`."] + pub fn roaring64_bitmap_remove_many( + r: *mut roaring64_bitmap_t, + n_args: usize, + vals: *const u64, + ); +} +extern "C" { + #[doc = " Remove all values in range [min, max)."] + pub fn roaring64_bitmap_remove_range(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " Remove all values in range [min, max]."] + pub fn roaring64_bitmap_remove_range_closed(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " Returns true if the provided value is present."] + pub fn roaring64_bitmap_contains(r: *const roaring64_bitmap_t, val: u64) -> bool; +} +extern "C" { + #[doc = " Returns true if all values in the range [min, max) are present."] + pub fn roaring64_bitmap_contains_range( + r: *const roaring64_bitmap_t, + min: u64, + max: u64, + ) -> bool; +} +extern "C" { + #[doc = " Check if an item is present using context from a previous insert or search\n for faster search.\n\n `context` will be used to store information between calls to make bulk\n operations faster. `*context` should be zero-initialized before the first\n call to this function.\n\n Modifying the bitmap in any way (other than `-bulk` suffixed functions)\n will invalidate the stored context, calling this function with a non-zero\n context after doing any modification invokes undefined behavior.\n\n In order to exploit this optimization, the caller should call this function\n with values with the same high 48 bits of the value consecutively."] + pub fn roaring64_bitmap_contains_bulk( + r: *const roaring64_bitmap_t, + context: *mut roaring64_bulk_context_t, + val: u64, + ) -> bool; +} +extern "C" { + #[doc = " Selects the element at index 'rank' where the smallest element is at index 0.\n If the size of the bitmap is strictly greater than rank, then this function\n returns true and sets element to the element of given rank. Otherwise, it\n returns false."] + pub fn roaring64_bitmap_select( + r: *const roaring64_bitmap_t, + rank: u64, + element: *mut u64, + ) -> bool; +} +extern "C" { + #[doc = " Returns the number of integers that are smaller or equal to x. Thus if x is\n the first element, this function will return 1. If x is smaller than the\n smallest element, this function will return 0.\n\n The indexing convention differs between roaring64_bitmap_select and\n roaring64_bitmap_rank: roaring_bitmap64_select refers to the smallest value\n as having index 0, whereas roaring64_bitmap_rank returns 1 when ranking\n the smallest value."] + pub fn roaring64_bitmap_rank(r: *const roaring64_bitmap_t, val: u64) -> u64; +} +extern "C" { + #[doc = " Returns true if the given value is in the bitmap, and sets `out_index` to the\n (0-based) index of the value in the bitmap. Returns false if the value is not\n in the bitmap."] + pub fn roaring64_bitmap_get_index( + r: *const roaring64_bitmap_t, + val: u64, + out_index: *mut u64, + ) -> bool; +} +extern "C" { + #[doc = " Returns the number of values in the bitmap."] + pub fn roaring64_bitmap_get_cardinality(r: *const roaring64_bitmap_t) -> u64; +} +extern "C" { + #[doc = " Returns the number of elements in the range [min, max)."] + pub fn roaring64_bitmap_range_cardinality( + r: *const roaring64_bitmap_t, + min: u64, + max: u64, + ) -> u64; +} +extern "C" { + #[doc = " Returns true if the bitmap is empty (cardinality is zero)."] + pub fn roaring64_bitmap_is_empty(r: *const roaring64_bitmap_t) -> bool; +} +extern "C" { + #[doc = " Returns the smallest value in the set, or UINT64_MAX if the set is empty."] + pub fn roaring64_bitmap_minimum(r: *const roaring64_bitmap_t) -> u64; +} +extern "C" { + #[doc = " Returns the largest value in the set, or 0 if empty."] + pub fn roaring64_bitmap_maximum(r: *const roaring64_bitmap_t) -> u64; +} +extern "C" { + #[doc = " Returns true if the result has at least one run container."] + pub fn roaring64_bitmap_run_optimize(r: *mut roaring64_bitmap_t) -> bool; +} +extern "C" { + #[doc = " Perform internal consistency checks.\n\n Returns true if the bitmap is consistent. It may be useful to call this\n after deserializing bitmaps from untrusted sources. If\n roaring64_bitmap_internal_validate returns true, then the bitmap is\n consistent and can be trusted not to cause crashes or memory corruption.\n\n If reason is non-null, it will be set to a string describing the first\n inconsistency found if any."] + pub fn roaring64_bitmap_internal_validate( + r: *const roaring64_bitmap_t, + reason: *mut *const ::std::os::raw::c_char, + ) -> bool; +} +extern "C" { + #[doc = " Return true if the two bitmaps contain the same elements."] + pub fn roaring64_bitmap_equals( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> bool; +} +extern "C" { + #[doc = " Return true if all the elements of r1 are also in r2."] + pub fn roaring64_bitmap_is_subset( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> bool; +} +extern "C" { + #[doc = " Return true if all the elements of r1 are also in r2, and r2 is strictly\n greater than r1."] + pub fn roaring64_bitmap_is_strict_subset( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> bool; +} +extern "C" { + #[doc = " Computes the intersection between two bitmaps and returns new bitmap. The\n caller is responsible for free-ing the result.\n\n Performance hint: if you are computing the intersection between several\n bitmaps, two-by-two, it is best to start with the smallest bitmaps. You may\n also rely on roaring64_bitmap_and_inplace to avoid creating many temporary\n bitmaps."] + pub fn roaring64_bitmap_and( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Computes the size of the intersection between two bitmaps."] + pub fn roaring64_bitmap_and_cardinality( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> u64; +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_and()`, modifies `r1`. `r1` and `r2`\n are allowed to be equal.\n\n Performance hint: if you are computing the intersection between several\n bitmaps, two-by-two, it is best to start with the smallest bitmaps."] + pub fn roaring64_bitmap_and_inplace(r1: *mut roaring64_bitmap_t, r2: *const roaring64_bitmap_t); +} +extern "C" { + #[doc = " Check whether two bitmaps intersect."] + pub fn roaring64_bitmap_intersect( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> bool; +} +extern "C" { + #[doc = " Check whether a bitmap intersects the range [min, max)."] + pub fn roaring64_bitmap_intersect_with_range( + r: *const roaring64_bitmap_t, + min: u64, + max: u64, + ) -> bool; +} +extern "C" { + #[doc = " Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto\n distance, or the Jaccard similarity coefficient)\n\n The Jaccard index is undefined if both bitmaps are empty."] + pub fn roaring64_bitmap_jaccard_index( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> f64; +} +extern "C" { + #[doc = " Computes the union between two bitmaps and returns new bitmap. The caller is\n responsible for free-ing the result."] + pub fn roaring64_bitmap_or( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Computes the size of the union between two bitmaps."] + pub fn roaring64_bitmap_or_cardinality( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> u64; +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_or(), modifies `r1`."] + pub fn roaring64_bitmap_or_inplace(r1: *mut roaring64_bitmap_t, r2: *const roaring64_bitmap_t); +} +extern "C" { + #[doc = " Computes the symmetric difference (xor) between two bitmaps and returns a new\n bitmap. The caller is responsible for free-ing the result."] + pub fn roaring64_bitmap_xor( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Computes the size of the symmetric difference (xor) between two bitmaps."] + pub fn roaring64_bitmap_xor_cardinality( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> u64; +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_xor()`, modifies `r1`. `r1` and `r2`\n are not allowed to be equal (that would result in an empty bitmap)."] + pub fn roaring64_bitmap_xor_inplace(r1: *mut roaring64_bitmap_t, r2: *const roaring64_bitmap_t); +} +extern "C" { + #[doc = " Computes the difference (andnot) between two bitmaps and returns a new\n bitmap. The caller is responsible for free-ing the result."] + pub fn roaring64_bitmap_andnot( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Computes the size of the difference (andnot) between two bitmaps."] + pub fn roaring64_bitmap_andnot_cardinality( + r1: *const roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ) -> u64; +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_andnot()`, modifies `r1`. `r1` and `r2`\n are not allowed to be equal (that would result in an empty bitmap)."] + pub fn roaring64_bitmap_andnot_inplace( + r1: *mut roaring64_bitmap_t, + r2: *const roaring64_bitmap_t, + ); +} +extern "C" { + #[doc = " Compute the negation of the bitmap in the interval [min, max).\n The number of negated values is `max - min`. Areas outside the range are\n passed through unchanged."] + pub fn roaring64_bitmap_flip( + r: *const roaring64_bitmap_t, + min: u64, + max: u64, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Compute the negation of the bitmap in the interval [min, max].\n The number of negated values is `max - min + 1`. Areas outside the range are\n passed through unchanged."] + pub fn roaring64_bitmap_flip_closed( + r: *const roaring64_bitmap_t, + min: u64, + max: u64, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_flip`. Compute the negation of the\n bitmap in the interval [min, max). The number of negated values is `max -\n min`. Areas outside the range are passed through unchanged."] + pub fn roaring64_bitmap_flip_inplace(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " In-place version of `roaring64_bitmap_flip_closed`. Compute the negation of\n the bitmap in the interval [min, max]. The number of negated values is `max -\n min + 1`. Areas outside the range are passed through unchanged."] + pub fn roaring64_bitmap_flip_closed_inplace(r: *mut roaring64_bitmap_t, min: u64, max: u64); +} +extern "C" { + #[doc = " How many bytes are required to serialize this bitmap.\n\n This is meant to be compatible with other languages:\n https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations"] + pub fn roaring64_bitmap_portable_size_in_bytes(r: *const roaring64_bitmap_t) -> usize; +} +extern "C" { + #[doc = " Write a bitmap to a buffer. The output buffer should refer to at least\n `roaring64_bitmap_portable_size_in_bytes(r)` bytes of allocated memory.\n\n Returns how many bytes were written, which should match\n `roaring64_bitmap_portable_size_in_bytes(r)`.\n\n This is meant to be compatible with other languages:\n https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] + pub fn roaring64_bitmap_portable_serialize( + r: *const roaring64_bitmap_t, + buf: *mut ::std::os::raw::c_char, + ) -> usize; +} +extern "C" { + #[doc = " Check how many bytes would be read (up to maxbytes) at this pointer if there\n is a valid bitmap, returns zero if there is no valid bitmap.\n\n This is meant to be compatible with other languages\n https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations"] + pub fn roaring64_bitmap_portable_deserialize_size( + buf: *const ::std::os::raw::c_char, + maxbytes: usize, + ) -> usize; +} +extern "C" { + #[doc = " Read a bitmap from a serialized buffer safely (reading up to maxbytes).\n In case of failure, NULL is returned.\n\n This is meant to be compatible with other languages\n https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations\n\n The function itself is safe in the sense that it will not cause buffer\n overflows. However, for correct operations, it is assumed that the bitmap\n read was once serialized from a valid bitmap (i.e., it follows the format\n specification). If you provided an incorrect input (garbage), then the bitmap\n read may not be in a valid state and following operations may not lead to\n sensible results. In particular, the serialized array containers need to be\n in sorted order, and the run containers should be in sorted non-overlapping\n order. This is is guaranteed to happen when serializing an existing bitmap,\n but not for random inputs.\n\n This function is endian-sensitive. If you have a big-endian system (e.g., a\n mainframe IBM s390x), the data format is going to be big-endian and not\n compatible with little-endian systems."] + pub fn roaring64_bitmap_portable_deserialize_safe( + buf: *const ::std::os::raw::c_char, + maxbytes: usize, + ) -> *mut roaring64_bitmap_t; +} +extern "C" { + #[doc = " Iterate over the bitmap elements. The function `iterator` is called once for\n all the values with `ptr` (can be NULL) as the second parameter of each call.\n\n `roaring_iterator64` is simply a pointer to a function that returns a bool\n and takes `(uint64_t, void*)` as inputs. True means that the iteration should\n continue, while false means that it should stop.\n\n Returns true if the `roaring64_iterator` returned true throughout (so that\n all data points were necessarily visited).\n\n Iteration is ordered from the smallest to the largest elements."] + pub fn roaring64_bitmap_iterate( + r: *const roaring64_bitmap_t, + iterator: roaring_iterator64, + ptr: *mut ::std::os::raw::c_void, + ) -> bool; +} +extern "C" { + #[doc = " Convert the bitmap to a sorted array `out`.\n\n Caller is responsible to ensure that there is enough memory allocated, e.g.\n ```\n out = malloc(roaring64_bitmap_get_cardinality(bitmap) * sizeof(uint64_t));\n ```"] + pub fn roaring64_bitmap_to_uint64_array(r: *const roaring64_bitmap_t, out: *mut u64); +} +extern "C" { + #[doc = " Create an iterator object that can be used to iterate through the values.\n Caller is responsible for calling `roaring64_iterator_free()`.\n\n The iterator is initialized. If there is a value, then this iterator points\n to the first value and `roaring64_iterator_has_value()` returns true. The\n value can be retrieved with `roaring64_iterator_value()`."] + pub fn roaring64_iterator_create(r: *const roaring64_bitmap_t) -> *mut roaring64_iterator_t; +} +extern "C" { + #[doc = " Create an iterator object that can be used to iterate through the values.\n Caller is responsible for calling `roaring64_iterator_free()`.\n\n The iterator is initialized. If there is a value, then this iterator points\n to the last value and `roaring64_iterator_has_value()` returns true. The\n value can be retrieved with `roaring64_iterator_value()`."] + pub fn roaring64_iterator_create_last( + r: *const roaring64_bitmap_t, + ) -> *mut roaring64_iterator_t; +} +extern "C" { + #[doc = " Re-initializes an existing iterator. Functionally the same as\n `roaring64_iterator_create` without a allocation."] + pub fn roaring64_iterator_reinit(r: *const roaring64_bitmap_t, it: *mut roaring64_iterator_t); +} +extern "C" { + #[doc = " Re-initializes an existing iterator. Functionally the same as\n `roaring64_iterator_create_last` without a allocation."] + pub fn roaring64_iterator_reinit_last( + r: *const roaring64_bitmap_t, + it: *mut roaring64_iterator_t, + ); +} +extern "C" { + #[doc = " Creates a copy of the iterator. Caller is responsible for calling\n `roaring64_iterator_free()` on the resulting iterator."] + pub fn roaring64_iterator_copy(it: *const roaring64_iterator_t) -> *mut roaring64_iterator_t; +} +extern "C" { + #[doc = " Free the iterator."] + pub fn roaring64_iterator_free(it: *mut roaring64_iterator_t); +} +extern "C" { + #[doc = " Returns true if the iterator currently points to a value. If so, calling\n `roaring64_iterator_value()` returns the value."] + pub fn roaring64_iterator_has_value(it: *const roaring64_iterator_t) -> bool; +} +extern "C" { + #[doc = " Returns the value the iterator currently points to. Should only be called if\n `roaring64_iterator_has_value()` returns true."] + pub fn roaring64_iterator_value(it: *const roaring64_iterator_t) -> u64; +} +extern "C" { + #[doc = " Advance the iterator. If there is a new value, then\n `roaring64_iterator_has_value()` returns true. Values are traversed in\n increasing order. For convenience, returns the result of\n `roaring64_iterator_has_value()`.\n\n Once this returns false, `roaring64_iterator_advance` should not be called on\n the iterator again. Calling `roaring64_iterator_previous` is allowed."] + pub fn roaring64_iterator_advance(it: *mut roaring64_iterator_t) -> bool; +} +extern "C" { + #[doc = " Decrement the iterator. If there is a new value, then\n `roaring64_iterator_has_value()` returns true. Values are traversed in\n decreasing order. For convenience, returns the result of\n `roaring64_iterator_has_value()`.\n\n Once this returns false, `roaring64_iterator_previous` should not be called\n on the iterator again. Calling `roaring64_iterator_advance` is allowed."] + pub fn roaring64_iterator_previous(it: *mut roaring64_iterator_t) -> bool; +} +extern "C" { + #[doc = " Move the iterator to the first value greater than or equal to `val`, if it\n exists at or after the current position of the iterator. If there is a new\n value, then `roaring64_iterator_has_value()` returns true. Values are\n traversed in increasing order. For convenience, returns the result of\n `roaring64_iterator_has_value()`."] + pub fn roaring64_iterator_move_equalorlarger(it: *mut roaring64_iterator_t, val: u64) -> bool; +} +extern "C" { + #[doc = " Reads up to `count` values from the iterator into the given `buf`. Returns\n the number of elements read. The number of elements read can be smaller than\n `count`, which means that there are no more elements in the bitmap.\n\n This function can be used together with other iterator functions."] + pub fn roaring64_iterator_read(it: *mut roaring64_iterator_t, buf: *mut u64, count: u64) + -> u64; +} diff --git a/croaring-sys/CRoaring/roaring.c b/croaring-sys/CRoaring/roaring.c index 72960e6..7385cff 100644 --- a/croaring-sys/CRoaring/roaring.c +++ b/croaring-sys/CRoaring/roaring.c @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2023-09-27T16:30:23Z +// Created by amalgamation.sh on 2024-04-02T13:42:32Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -65,10 +65,7 @@ /* begin file include/roaring/isadetection.h */ #ifndef ROARING_ISADETECTION_H #define ROARING_ISADETECTION_H -#if defined(__x86_64__) || defined(_M_AMD64) // x64 - - - +#if defined(__x86_64__) || defined(_M_AMD64) // x64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #ifdef __has_include @@ -76,35 +73,38 @@ // fully supporting AVX-512. #if __has_include() #define CROARING_COMPILER_SUPPORTS_AVX512 1 -#endif // #if __has_include() -#endif // #ifdef __has_include +#endif // #if __has_include() +#endif // #ifdef __has_include // Visual Studio 2019 and up support AVX-512 #ifdef _MSC_VER #if _MSC_VER >= 1920 #define CROARING_COMPILER_SUPPORTS_AVX512 1 -#endif // #if _MSC_VER >= 1920 -#endif // #ifdef _MSC_VER +#endif // #if _MSC_VER >= 1920 +#endif // #ifdef _MSC_VER #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #define CROARING_COMPILER_SUPPORTS_AVX512 0 -#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 -#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 - +#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif enum { - ROARING_SUPPORTS_AVX2 = 1, - ROARING_SUPPORTS_AVX512 = 2, + ROARING_SUPPORTS_AVX2 = 1, + ROARING_SUPPORTS_AVX512 = 2, }; int croaring_hardware_support(void); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif -#endif // x64 -#endif // ROARING_ISADETECTION_H +#endif // x64 +#endif // ROARING_ISADETECTION_H /* end file include/roaring/isadetection.h */ /* begin file include/roaring/containers/perfparameters.h */ #ifndef PERFPARAMETERS_H_ @@ -113,7 +113,9 @@ int croaring_hardware_support(void); #include #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /** @@ -148,7 +150,9 @@ enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; #endif #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif @@ -166,16 +170,16 @@ enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; #define INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ #ifdef __cplusplus - #include // used by casting helper for compile-time check +#include // used by casting helper for compile-time check #endif // The preferences are a separate file to separate out tweakable parameters #ifdef __cplusplus -namespace roaring { namespace internal { // No extern "C" (contains template) +namespace roaring { +namespace internal { // No extern "C" (contains template) #endif - /* * Since roaring_array_t's definition is not opaque, the container type is * part of the API. If it's not going to be `void*` then it needs a name, and @@ -189,7 +193,6 @@ namespace roaring { namespace internal { // No extern "C" (contains template) typedef ROARING_CONTAINER_T container_t; #undef ROARING_CONTAINER_T - /* * See ROARING_CONTAINER_T for notes on using container_t as a base class. * This macro helps make the following pattern look nicer: @@ -205,14 +208,11 @@ typedef ROARING_CONTAINER_T container_t; * } */ #if defined(__cplusplus) - #define STRUCT_CONTAINER(name) \ - struct name : public container_t /* { ... } */ +#define STRUCT_CONTAINER(name) struct name : public container_t /* { ... } */ #else - #define STRUCT_CONTAINER(name) \ - struct name /* { ... } */ +#define STRUCT_CONTAINER(name) struct name /* { ... } */ #endif - /** * Since container_t* is not void* in C++, "dangerous" casts are not needed to * downcast; only a static_cast<> is needed. Define a macro for static casting @@ -231,38 +231,36 @@ typedef ROARING_CONTAINER_T container_t; * leveraging to make sure it's legal in the C++ build. */ #ifdef __cplusplus - #define CAST(type,value) static_cast(value) - #define movable_CAST(type,value) movable_CAST_HELPER(value) - - template - PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { - typedef typename std::remove_pointer::type PDerived; - typedef typename std::remove_pointer::type Derived; - static_assert( - std::is_base_of::value, - "use movable_CAST() for container_t** => xxx_container_t**" - ); - return reinterpret_cast(ptr_to_ptr); - } +#define CAST(type, value) static_cast(value) +#define movable_CAST(type, value) movable_CAST_HELPER(value) + +template +PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { + typedef typename std::remove_pointer::type PDerived; + typedef typename std::remove_pointer::type Derived; + static_assert(std::is_base_of::value, + "use movable_CAST() for container_t** => xxx_container_t**"); + return reinterpret_cast(ptr_to_ptr); +} #else - #define CAST(type,value) ((type)value) - #define movable_CAST(type, value) ((type)value) +#define CAST(type, value) ((type)value) +#define movable_CAST(type, value) ((type)value) #endif // Use for converting e.g. an `array_container_t**` to a `container_t**` // -#define movable_CAST_base(c) movable_CAST(container_t **, c) - +#define movable_CAST_base(c) movable_CAST(container_t **, c) #ifdef __cplusplus -} } // namespace roaring { namespace internal { +} +} // namespace roaring { namespace internal { #endif -#endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ +#endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ /* end file include/roaring/containers/container_defs.h */ /* begin file include/roaring/array_util.h */ -#ifndef ARRAY_UTIL_H -#define ARRAY_UTIL_H +#ifndef CROARING_ARRAY_UTIL_H +#define CROARING_ARRAY_UTIL_H #include // for size_t #include @@ -271,11 +269,17 @@ typedef ROARING_CONTAINER_T container_t; #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif - #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* @@ -283,8 +287,8 @@ extern "C" { namespace roaring { namespace internal { * Assumes that array is sorted, has logarithmic complexity. * if the result is x, then: * if ( x>0 ) you have array[x] = ikey - * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that array[-x-1]=ikey) - * keys the array sorted. + * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that + * array[-x-1]=ikey) keys the array sorted. */ inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, uint16_t ikey) { @@ -307,9 +311,9 @@ inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, /** * Galloping search * Assumes that array is sorted, has logarithmic complexity. - * if the result is x, then if x = length, you have that all values in array between pos and length - * are smaller than min. - * otherwise returns the first index x such that array[x] >= min. + * if the result is x, then if x = length, you have that all values in array + * between pos and length are smaller than min. otherwise returns the first + * index x such that array[x] >= min. */ static inline int32_t advanceUntil(const uint16_t *array, int32_t pos, int32_t length, uint16_t min) { @@ -364,7 +368,7 @@ static inline int32_t count_less(const uint16_t *array, int32_t lenarray, uint16_t ikey) { if (lenarray == 0) return 0; int32_t pos = binarySearch(array, lenarray, ikey); - return pos >= 0 ? pos : -(pos+1); + return pos >= 0 ? pos : -(pos + 1); } /** @@ -376,9 +380,9 @@ static inline int32_t count_greater(const uint16_t *array, int32_t lenarray, if (lenarray == 0) return 0; int32_t pos = binarySearch(array, lenarray, ikey); if (pos >= 0) { - return lenarray - (pos+1); + return lenarray - (pos + 1); } else { - return lenarray - (-pos-1); + return lenarray - (-pos - 1); } } @@ -394,17 +398,17 @@ int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, uint16_t *C); int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b); + const uint16_t *__restrict__ B, size_t s_b); /** * Take an array container and write it out to a 32-bit array, using base * as the offset. */ -int array_container_to_uint32_array_vector16(void *vout, const uint16_t* array, size_t cardinality, - uint32_t base); +int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base); #if CROARING_COMPILER_SUPPORTS_AVX512 -int avx512_array_container_to_uint32_array(void *vout, const uint16_t* array, size_t cardinality, - uint32_t base); +int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base); #endif /** * Compute the cardinality of the intersection using SSE4 instructions @@ -427,10 +431,11 @@ int32_t intersect_skewed_uint16_cardinality(const uint16_t *smallarray, const uint16_t *largearray, size_t size_l); - -/* Check whether the size of the intersection between one small and one large set of uint16_t is non-zero. */ +/* Check whether the size of the intersection between one small and one large + * set of uint16_t is non-zero. */ bool intersect_skewed_uint16_nonempty(const uint16_t *smallarray, size_t size_s, - const uint16_t *largearray, size_t size_l); + const uint16_t *largearray, + size_t size_l); /** * Generic intersection function. */ @@ -446,7 +451,7 @@ int32_t intersect_uint16_cardinality(const uint16_t *A, const size_t lenA, * Checking whether the size of the intersection is non-zero. */ bool intersect_uint16_nonempty(const uint16_t *A, const size_t lenA, - const uint16_t *B, const size_t lenB); + const uint16_t *B, const size_t lenB); /** * Generic union function. */ @@ -510,18 +515,22 @@ size_t union_uint32_card(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, size_t size_2); /** -* combines union_uint16 and union_vector16 optimally -*/ -size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, - size_t size_2, uint16_t *buffer); - + * combines union_uint16 and union_vector16 optimally + */ +size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, + const uint16_t *set_2, size_t size_2, + uint16_t *buffer); bool memequals(const void *s1, const void *s2, size_t n); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop #endif - #endif /* end file include/roaring/array_util.h */ /* begin file include/roaring/utilasm.h */ @@ -535,7 +544,8 @@ bool memequals(const void *s1, const void *s2, size_t n); #ifdef __cplusplus -extern "C" { namespace roaring { +extern "C" { +namespace roaring { #endif #if defined(CROARING_INLINE_ASM) @@ -547,14 +557,14 @@ extern "C" { namespace roaring { : /* write */ \ "r"(bitsReg), /* read only */ \ "r"(srcReg) /* read only */ \ - ) + ) #define ASM_INPLACESHIFT_RIGHT(srcReg, bitsReg) \ __asm volatile("shrx %1, %0, %0" \ : "+r"(srcReg) \ : /* read/write */ \ "r"(bitsReg) /* read only */ \ - ) + ) #define ASM_SHIFT_LEFT(srcReg, bitsReg, destReg) \ __asm volatile("shlx %1, %2, %0" \ @@ -562,7 +572,7 @@ extern "C" { namespace roaring { : /* write */ \ "r"(bitsReg), /* read only */ \ "r"(srcReg) /* read only */ \ - ) + ) // set bit at position testBit within testByte to 1 and // copy cmovDst to cmovSrc if that bit was previously clear #define ASM_SET_BIT_INC_WAS_CLEAR(testByte, testBit, count) \ @@ -573,7 +583,7 @@ extern "C" { namespace roaring { "+r"(count) \ : /* read/write */ \ "r"(testBit) /* read only */ \ - ) + ) #define ASM_CLEAR_BIT_DEC_WAS_SET(testByte, testBit, count) \ __asm volatile( \ @@ -583,7 +593,7 @@ extern "C" { namespace roaring { "+r"(count) \ : /* read/write */ \ "r"(testBit) /* read only */ \ - ) + ) #define ASM_BT64(testByte, testBit, count) \ __asm volatile( \ @@ -593,19 +603,20 @@ extern "C" { namespace roaring { : /* write */ \ "r"(testByte), /* read only */ \ "r"(testBit) /* read only */ \ - ) + ) #endif #ifdef __cplusplus -} } // extern "C" { namespace roaring { +} +} // extern "C" { namespace roaring { #endif -#endif /* INCLUDE_UTILASM_H_ */ +#endif /* INCLUDE_UTILASM_H_ */ /* end file include/roaring/utilasm.h */ /* begin file include/roaring/bitset_util.h */ -#ifndef BITSET_UTIL_H -#define BITSET_UTIL_H +#ifndef CROARING_BITSET_UTIL_H +#define CROARING_BITSET_UTIL_H #include @@ -613,11 +624,17 @@ extern "C" { namespace roaring { #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif - #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* @@ -630,7 +647,7 @@ static inline void bitset_set_range(uint64_t *words, uint32_t start, uint32_t endword = (end - 1) / 64; if (firstword == endword) { words[firstword] |= ((~UINT64_C(0)) << (start % 64)) & - ((~UINT64_C(0)) >> ((~end + 1) % 64)); + ((~UINT64_C(0)) >> ((~end + 1) % 64)); return; } words[firstword] |= (~UINT64_C(0)) << (start % 64); @@ -640,7 +657,6 @@ static inline void bitset_set_range(uint64_t *words, uint32_t start, words[endword] |= (~UINT64_C(0)) >> ((~end + 1) % 64); } - /* * Find the cardinality of the bitset in [begin,begin+lenminusone] */ @@ -651,16 +667,17 @@ static inline int bitset_lenrange_cardinality(const uint64_t *words, uint32_t endword = (start + lenminusone) / 64; if (firstword == endword) { return roaring_hamming(words[firstword] & - ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) - << (start % 64)); + ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) + << (start % 64)); } - int answer = roaring_hamming(words[firstword] & ((~UINT64_C(0)) << (start % 64))); + int answer = + roaring_hamming(words[firstword] & ((~UINT64_C(0)) << (start % 64))); for (uint32_t i = firstword + 1; i < endword; i++) { answer += roaring_hamming(words[i]); } - answer += - roaring_hamming(words[endword] & - (~UINT64_C(0)) >> (((~start + 1) - lenminusone - 1) % 64)); + answer += roaring_hamming(words[endword] & + (~UINT64_C(0)) >> + (((~start + 1) - lenminusone - 1) % 64)); return answer; } @@ -673,9 +690,9 @@ static inline bool bitset_lenrange_empty(const uint64_t *words, uint32_t start, uint32_t endword = (start + lenminusone) / 64; if (firstword == endword) { return (words[firstword] & ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) - << (start % 64)) == 0; + << (start % 64)) == 0; } - if (((words[firstword] & ((~UINT64_C(0)) << (start%64)))) != 0) { + if (((words[firstword] & ((~UINT64_C(0)) << (start % 64)))) != 0) { return false; } for (uint32_t i = firstword + 1; i < endword; i++) { @@ -683,13 +700,13 @@ static inline bool bitset_lenrange_empty(const uint64_t *words, uint32_t start, return false; } } - if ((words[endword] & (~UINT64_C(0)) >> (((~start + 1) - lenminusone - 1) % 64)) != 0) { + if ((words[endword] & + (~UINT64_C(0)) >> (((~start + 1) - lenminusone - 1) % 64)) != 0) { return false; } return true; } - /* * Set all bits in indexes [begin,begin+lenminusone] to true. */ @@ -699,7 +716,7 @@ static inline void bitset_set_lenrange(uint64_t *words, uint32_t start, uint32_t endword = (start + lenminusone) / 64; if (firstword == endword) { words[firstword] |= ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) - << (start % 64); + << (start % 64); return; } uint64_t temp = words[endword]; @@ -735,7 +752,7 @@ static inline void bitset_reset_range(uint64_t *words, uint32_t start, uint32_t endword = (end - 1) / 64; if (firstword == endword) { words[firstword] &= ~(((~UINT64_C(0)) << (start % 64)) & - ((~UINT64_C(0)) >> ((~end + 1) % 64))); + ((~UINT64_C(0)) >> ((~end + 1) % 64))); return; } words[firstword] &= ~((~UINT64_C(0)) << (start % 64)); @@ -764,9 +781,9 @@ size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, uint32_t *out, size_t outcapacity, uint32_t base); -size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, - uint32_t *out, size_t outcapacity, - uint32_t base); +size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, + uint32_t *out, size_t outcapacity, + uint32_t base); /* * Given a bitset containing "length" 64-bit words, write out the position * of all the set bits to "out", values start at "base". @@ -799,9 +816,9 @@ size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, uint16_t *out, size_t outcapacity, uint16_t base); -size_t bitset_extract_setbits_avx512_uint16(const uint64_t *words, size_t length, - uint16_t *out, size_t outcapacity, - uint16_t base); +size_t bitset_extract_setbits_avx512_uint16(const uint64_t *words, + size_t length, uint16_t *out, + size_t outcapacity, uint16_t base); /* * Given a bitset containing "length" 64-bit words, write out the position @@ -826,10 +843,9 @@ size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, * * Returns how many values were actually decoded. */ -size_t bitset_extract_intersection_setbits_uint16(const uint64_t * __restrict__ words1, - const uint64_t * __restrict__ words2, - size_t length, uint16_t *out, - uint16_t base); +size_t bitset_extract_intersection_setbits_uint16( + const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, + size_t length, uint16_t *out, uint16_t base); /* * Given a bitset having cardinality card, set all bit values in the list (there @@ -982,7 +998,7 @@ inline static uint64_t avx2_harley_seal_popcount256(const __m256i *data, } CROARING_UNTARGET_AVX2 -#define AVXPOPCNTFNC(opname, avx_intrinsic) \ +#define CROARING_AVXPOPCNTFNC(opname, avx_intrinsic) \ static inline uint64_t avx2_harley_seal_popcount256_##opname( \ const __m256i *data1, const __m256i *data2, const uint64_t size) { \ __m256i total = _mm256_setzero_si256(); \ @@ -1163,146 +1179,141 @@ CROARING_UNTARGET_AVX2 } CROARING_TARGET_AVX2 -AVXPOPCNTFNC(or, _mm256_or_si256) +CROARING_AVXPOPCNTFNC(or, _mm256_or_si256) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVXPOPCNTFNC(union, _mm256_or_si256) +CROARING_AVXPOPCNTFNC(union, _mm256_or_si256) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVXPOPCNTFNC(and, _mm256_and_si256) +CROARING_AVXPOPCNTFNC(and, _mm256_and_si256) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVXPOPCNTFNC(intersection, _mm256_and_si256) +CROARING_AVXPOPCNTFNC(intersection, _mm256_and_si256) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVXPOPCNTFNC (xor, _mm256_xor_si256) +CROARING_AVXPOPCNTFNC(xor, _mm256_xor_si256) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVXPOPCNTFNC(andnot, _mm256_andnot_si256) +CROARING_AVXPOPCNTFNC(andnot, _mm256_andnot_si256) CROARING_UNTARGET_AVX2 - -#define VPOPCNT_AND_ADD(ptr, i, accu) \ - const __m512i v##i = _mm512_loadu_si512((const __m512i*)ptr + i); \ - const __m512i p##i = _mm512_popcnt_epi64(v##i); \ - accu = _mm512_add_epi64(accu, p##i); +#define VPOPCNT_AND_ADD(ptr, i, accu) \ + const __m512i v##i = _mm512_loadu_si512((const __m512i *)ptr + i); \ + const __m512i p##i = _mm512_popcnt_epi64(v##i); \ + accu = _mm512_add_epi64(accu, p##i); #if CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX512 static inline uint64_t sum_epu64_256(const __m256i v) { - - return (uint64_t)(_mm256_extract_epi64(v, 0)) - + (uint64_t)(_mm256_extract_epi64(v, 1)) - + (uint64_t)(_mm256_extract_epi64(v, 2)) - + (uint64_t)(_mm256_extract_epi64(v, 3)); + return (uint64_t)(_mm256_extract_epi64(v, 0)) + + (uint64_t)(_mm256_extract_epi64(v, 1)) + + (uint64_t)(_mm256_extract_epi64(v, 2)) + + (uint64_t)(_mm256_extract_epi64(v, 3)); } - static inline uint64_t simd_sum_epu64(const __m512i v) { - - __m256i lo = _mm512_extracti64x4_epi64(v, 0); - __m256i hi = _mm512_extracti64x4_epi64(v, 1); + __m256i lo = _mm512_extracti64x4_epi64(v, 0); + __m256i hi = _mm512_extracti64x4_epi64(v, 1); return sum_epu64_256(lo) + sum_epu64_256(hi); } -static inline uint64_t avx512_vpopcount(const __m512i* data, const uint64_t size) -{ +static inline uint64_t avx512_vpopcount(const __m512i *data, + const uint64_t size) { const uint64_t limit = size - size % 4; __m512i total = _mm512_setzero_si512(); uint64_t i = 0; - for (; i < limit; i += 4) - { + for (; i < limit; i += 4) { VPOPCNT_AND_ADD(data + i, 0, total); VPOPCNT_AND_ADD(data + i, 1, total); VPOPCNT_AND_ADD(data + i, 2, total); VPOPCNT_AND_ADD(data + i, 3, total); } - - for (; i < size; i++) - { - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(_mm512_loadu_si512(data + i))); + + for (; i < size; i++) { + total = _mm512_add_epi64( + total, _mm512_popcnt_epi64(_mm512_loadu_si512(data + i))); } - + return simd_sum_epu64(total); } CROARING_UNTARGET_AVX512 #endif -#define AVXPOPCNTFNC512(opname, avx_intrinsic) \ - static inline uint64_t avx512_harley_seal_popcount512_##opname( \ - const __m512i *data1, const __m512i *data2, const uint64_t size) { \ - __m512i total = _mm512_setzero_si512(); \ - const uint64_t limit = size - size % 4; \ - uint64_t i = 0; \ - for (; i < limit; i += 4) { \ - __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ - _mm512_loadu_si512(data2 + i)); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ - __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ - _mm512_loadu_si512(data2 + i + 1)); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ - __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ - _mm512_loadu_si512(data2 + i + 2)); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ - __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ - _mm512_loadu_si512(data2 + i + 3)); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ - } \ - for(; i < size; i++) { \ - __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ - _mm512_loadu_si512(data2 + i)); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ - } \ - return simd_sum_epu64(total); \ - } \ - static inline uint64_t avx512_harley_seal_popcount512andstore_##opname( \ - const __m512i *__restrict__ data1, const __m512i *__restrict__ data2, \ - __m512i *__restrict__ out, const uint64_t size) { \ - __m512i total = _mm512_setzero_si512(); \ - const uint64_t limit = size - size % 4; \ - uint64_t i = 0; \ - for (; i < limit; i += 4) { \ - __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ - _mm512_loadu_si512(data2 + i)); \ - _mm512_storeu_si512(out + i, a1); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ - __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ - _mm512_loadu_si512(data2 + i + 1)); \ - _mm512_storeu_si512(out + i + 1, a2); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ - __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ - _mm512_loadu_si512(data2 + i + 2)); \ - _mm512_storeu_si512(out + i + 2, a3); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ - __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ - _mm512_loadu_si512(data2 + i + 3)); \ - _mm512_storeu_si512(out + i + 3, a4); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ - } \ - for(; i < size; i++) { \ - __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ - _mm512_loadu_si512(data2 + i)); \ - _mm512_storeu_si512(out + i, a); \ - total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ - } \ - return simd_sum_epu64(total); \ - } \ +#define CROARING_AVXPOPCNTFNC512(opname, avx_intrinsic) \ + static inline uint64_t avx512_harley_seal_popcount512_##opname( \ + const __m512i *data1, const __m512i *data2, const uint64_t size) { \ + __m512i total = _mm512_setzero_si512(); \ + const uint64_t limit = size - size % 4; \ + uint64_t i = 0; \ + for (; i < limit; i += 4) { \ + __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ + __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ + _mm512_loadu_si512(data2 + i + 1)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ + __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ + _mm512_loadu_si512(data2 + i + 2)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ + __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ + _mm512_loadu_si512(data2 + i + 3)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ + } \ + for (; i < size; i++) { \ + __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ + } \ + return simd_sum_epu64(total); \ + } \ + static inline uint64_t avx512_harley_seal_popcount512andstore_##opname( \ + const __m512i *__restrict__ data1, const __m512i *__restrict__ data2, \ + __m512i *__restrict__ out, const uint64_t size) { \ + __m512i total = _mm512_setzero_si512(); \ + const uint64_t limit = size - size % 4; \ + uint64_t i = 0; \ + for (; i < limit; i += 4) { \ + __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + _mm512_storeu_si512(out + i, a1); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ + __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ + _mm512_loadu_si512(data2 + i + 1)); \ + _mm512_storeu_si512(out + i + 1, a2); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ + __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ + _mm512_loadu_si512(data2 + i + 2)); \ + _mm512_storeu_si512(out + i + 2, a3); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ + __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ + _mm512_loadu_si512(data2 + i + 3)); \ + _mm512_storeu_si512(out + i + 3, a4); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ + } \ + for (; i < size; i++) { \ + __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + _mm512_storeu_si512(out + i, a); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ + } \ + return simd_sum_epu64(total); \ + } #if CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX512 -AVXPOPCNTFNC512(or, _mm512_or_si512) -AVXPOPCNTFNC512(union, _mm512_or_si512) -AVXPOPCNTFNC512(and, _mm512_and_si512) -AVXPOPCNTFNC512(intersection, _mm512_and_si512) -AVXPOPCNTFNC512(xor, _mm512_xor_si512) -AVXPOPCNTFNC512(andnot, _mm512_andnot_si512) +CROARING_AVXPOPCNTFNC512(or, _mm512_or_si512) +CROARING_AVXPOPCNTFNC512(union, _mm512_or_si512) +CROARING_AVXPOPCNTFNC512(and, _mm512_and_si512) +CROARING_AVXPOPCNTFNC512(intersection, _mm512_and_si512) +CROARING_AVXPOPCNTFNC512(xor, _mm512_xor_si512) +CROARING_AVXPOPCNTFNC512(andnot, _mm512_andnot_si512) CROARING_UNTARGET_AVX512 #endif /*** @@ -1312,9 +1323,13 @@ CROARING_UNTARGET_AVX512 #endif // CROARING_IS_X64 #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal +} +} +} // extern "C" { namespace roaring { namespace internal +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop #endif - #endif /* end file include/roaring/bitset_util.h */ /* begin file include/roaring/containers/array.h */ @@ -1329,9 +1344,11 @@ CROARING_UNTARGET_AVX512 #include +// Include other headers after roaring_types.h #ifdef __cplusplus -extern "C" { namespace roaring { +extern "C" { +namespace roaring { // Note: in pure C++ code, you should avoid putting `using` in header files using api::roaring_iterator; @@ -1357,8 +1374,8 @@ STRUCT_CONTAINER(array_container_s) { typedef struct array_container_s array_container_t; -#define CAST_array(c) CAST(array_container_t *, c) // safer downcast -#define const_CAST_array(c) CAST(const array_container_t *, c) +#define CAST_array(c) CAST(array_container_t *, c) // safer downcast +#define const_CAST_array(c) CAST(const array_container_t *, c) #define movable_CAST_array(c) movable_CAST(array_container_t **, c) /* Create a new array with default. Return NULL in case of failure. See also @@ -1370,7 +1387,7 @@ array_container_t *array_container_create(void); array_container_t *array_container_create_given_capacity(int32_t size); /* Create a new array containing all values in [min,max). */ -array_container_t * array_container_create_range(uint32_t min, uint32_t max); +array_container_t *array_container_create_range(uint32_t min, uint32_t max); /* * Shrink the capacity to the actual size, return the number of bytes saved. @@ -1403,18 +1420,16 @@ void array_container_copy(const array_container_t *src, array_container_t *dst); void array_container_add_from_range(array_container_t *arr, uint32_t min, uint32_t max, uint16_t step); - static inline bool array_container_empty(const array_container_t *array) { return array->cardinality == 0; } /* check whether the cardinality is equal to the capacity (this does not mean -* that it contains 1<<16 elements) */ + * that it contains 1<<16 elements) */ static inline bool array_container_full(const array_container_t *array) { return array->cardinality == array->capacity; } - /* Compute the union of `src_1' and `src_2' and write the result to `dst' * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ void array_container_union(const array_container_t *src_1, @@ -1434,8 +1449,7 @@ void array_container_intersection(const array_container_t *src_1, /* Check whether src_1 and src_2 intersect. */ bool array_container_intersect(const array_container_t *src_1, - const array_container_t *src_2); - + const array_container_t *src_2); /* computers the size of the intersection between two arrays. */ @@ -1525,6 +1539,7 @@ int32_t array_container_read(int32_t cardinality, array_container_t *container, * that the cardinality of the container is already known. * */ +ALLOW_UNALIGNED static inline int32_t array_container_size_in_bytes( const array_container_t *container) { return container->cardinality * sizeof(uint16_t); @@ -1534,14 +1549,13 @@ static inline int32_t array_container_size_in_bytes( * Return true if the two arrays have the same content. */ ALLOW_UNALIGNED -static inline bool array_container_equals( - const array_container_t *container1, - const array_container_t *container2) { - +static inline bool array_container_equals(const array_container_t *container1, + const array_container_t *container2) { if (container1->cardinality != container2->cardinality) { return false; } - return memequals(container1->array, container2->array, container1->cardinality*2); + return memequals(container1->array, container2->array, + container1->cardinality * 2); } /** @@ -1597,7 +1611,8 @@ static inline void array_container_append(array_container_t *arr, * 0 -- value was already present * -1 -- value was not added because cardinality would exceed max_cardinality */ -static inline int array_container_try_add(array_container_t *arr, uint16_t value, +static inline int array_container_try_add(array_container_t *arr, + uint16_t value, int32_t max_cardinality) { const int32_t cardinality = arr->cardinality; @@ -1652,11 +1667,11 @@ inline bool array_container_contains(const array_container_t *arr, // return binarySearch(arr->array, arr->cardinality, pos) >= 0; // binary search with fallback to linear search for short ranges int32_t low = 0; - const uint16_t * carr = (const uint16_t *) arr->array; + const uint16_t *carr = (const uint16_t *)arr->array; int32_t high = arr->cardinality - 1; // while (high - low >= 0) { - while(high >= low + 16) { - int32_t middleIndex = (low + high)>>1; + while (high >= low + 16) { + int32_t middleIndex = (low + high) >> 1; uint16_t middleValue = carr[middleIndex]; if (middleValue < pos) { low = middleIndex + 1; @@ -1667,27 +1682,27 @@ inline bool array_container_contains(const array_container_t *arr, } } - for (int i=low; i <= high; i++) { + for (int i = low; i <= high; i++) { uint16_t v = carr[i]; if (v == pos) { return true; } - if ( v > pos ) return false; + if (v > pos) return false; } return false; - } -void array_container_offset(const array_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset); +void array_container_offset(const array_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); -//* Check whether a range of values from range_start (included) to range_end (excluded) is present. */ +//* Check whether a range of values from range_start (included) to range_end +//(excluded) is present. */ static inline bool array_container_contains_range(const array_container_t *arr, - uint32_t range_start, uint32_t range_end) { + uint32_t range_start, + uint32_t range_end) { const int32_t range_count = range_end - range_start; - const uint16_t rs_included = range_start; - const uint16_t re_included = range_end - 1; + const uint16_t rs_included = (uint16_t)range_start; + const uint16_t re_included = (uint16_t)(range_end - 1); // Empty range is always included if (range_count <= 0) { @@ -1697,10 +1712,12 @@ static inline bool array_container_contains_range(const array_container_t *arr, return false; } - const int32_t start = binarySearch(arr->array, arr->cardinality, rs_included); + const int32_t start = + binarySearch(arr->array, arr->cardinality, rs_included); // If this sorted array contains all items in the range: // * the start item must be found - // * the last item in range range_count must exist, and be the expected end value + // * the last item in range range_count must exist, and be the expected end + // value return (start >= 0) && (arr->cardinality >= start + range_count) && (arr->array[start + range_count - 1] == re_included); } @@ -1728,6 +1745,33 @@ inline int array_container_rank(const array_container_t *arr, uint16_t x) { } } +/* bulk version of array_container_rank(); return number of consumed elements + */ +inline uint32_t array_container_rank_many(const array_container_t *arr, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + const uint16_t high = (uint16_t)((*begin) >> 16); + uint32_t pos = 0; + const uint32_t *iter = begin; + for (; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if (xhigh != high) return iter - begin; // stop at next container + + const int32_t idx = + binarySearch(arr->array + pos, arr->cardinality - pos, (uint16_t)x); + const bool is_present = idx >= 0; + if (is_present) { + *(ans++) = start_rank + pos + (idx + 1); + pos = idx + 1; + } else { + *(ans++) = start_rank + pos + (-idx - 1); + } + } + return iter - begin; +} + /* Returns the index of x , if not exsist return -1 */ inline int array_container_get_index(const array_container_t *arr, uint16_t x) { const int32_t idx = binarySearch(arr->array, arr->cardinality, x); @@ -1740,14 +1784,15 @@ inline int array_container_get_index(const array_container_t *arr, uint16_t x) { } /* Returns the index of the first value equal or larger than x, or -1 */ -inline int array_container_index_equalorlarger(const array_container_t *arr, uint16_t x) { +inline int array_container_index_equalorlarger(const array_container_t *arr, + uint16_t x) { const int32_t idx = binarySearch(arr->array, arr->cardinality, x); const bool is_present = idx >= 0; if (is_present) { return idx; } else { - int32_t candidate = - idx - 1; - if(candidate < arr->cardinality) return candidate; + int32_t candidate = -idx - 1; + if (candidate < arr->cardinality) return candidate; return -1; } } @@ -1769,7 +1814,7 @@ static inline void array_container_add_range_nvals(array_container_t *array, &(array->array[array->cardinality - nvals_greater]), nvals_greater * sizeof(uint16_t)); for (uint32_t i = 0; i <= max - min; i++) { - array->array[nvals_less + i] = min + i; + array->array[nvals_less + i] = (uint16_t)(min + i); } array->cardinality = union_cardinality; } @@ -1780,9 +1825,10 @@ static inline void array_container_add_range_nvals(array_container_t *array, */ /*static inline void array_container_add_range(array_container_t *array, uint32_t min, uint32_t max) { - int32_t nvals_greater = count_greater(array->array, array->cardinality, max); - int32_t nvals_less = count_less(array->array, array->cardinality - nvals_greater, min); - array_container_add_range_nvals(array, min, max, nvals_less, nvals_greater); + int32_t nvals_greater = count_greater(array->array, array->cardinality, +max); int32_t nvals_less = count_less(array->array, array->cardinality - +nvals_greater, min); array_container_add_range_nvals(array, min, max, +nvals_less, nvals_greater); }*/ /* @@ -1790,15 +1836,17 @@ static inline void array_container_add_range_nvals(array_container_t *array, */ static inline void array_container_remove_range(array_container_t *array, uint32_t pos, uint32_t count) { - if (count != 0) { - memmove(&(array->array[pos]), &(array->array[pos+count]), - (array->cardinality - pos - count) * sizeof(uint16_t)); - array->cardinality -= count; - } + if (count != 0) { + memmove(&(array->array[pos]), &(array->array[pos + count]), + (array->cardinality - pos - count) * sizeof(uint16_t)); + array->cardinality -= count; + } } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_ARRAY_H_ */ @@ -1816,9 +1864,11 @@ static inline void array_container_remove_range(array_container_t *array, #include +// Include other headers after roaring_types.h #ifdef __cplusplus -extern "C" { namespace roaring { +extern "C" { +namespace roaring { // Note: in pure C++ code, you should avoid putting `using` in header files using api::roaring_iterator; @@ -1827,8 +1877,6 @@ using api::roaring_iterator64; namespace internal { #endif - - enum { BITSET_CONTAINER_SIZE_IN_WORDS = (1 << 16) / 64, BITSET_UNKNOWN_CARDINALITY = -1 @@ -1841,8 +1889,8 @@ STRUCT_CONTAINER(bitset_container_s) { typedef struct bitset_container_s bitset_container_t; -#define CAST_bitset(c) CAST(bitset_container_t *, c) // safer downcast -#define const_CAST_bitset(c) CAST(const bitset_container_t *, c) +#define CAST_bitset(c) CAST(bitset_container_t *, c) // safer downcast +#define const_CAST_bitset(c) CAST(const bitset_container_t *, c) #define movable_CAST_bitset(c) movable_CAST(bitset_container_t **, c) /* Create a new bitset. Return NULL in case of failure. */ @@ -1990,28 +2038,29 @@ inline bool bitset_container_get(const bitset_container_t *bitset, #endif /* -* Check if all bits are set in a range of positions from pos_start (included) to -* pos_end (excluded). -*/ + * Check if all bits are set in a range of positions from pos_start (included) + * to pos_end (excluded). + */ static inline bool bitset_container_get_range(const bitset_container_t *bitset, - uint32_t pos_start, uint32_t pos_end) { - + uint32_t pos_start, + uint32_t pos_end) { const uint32_t start = pos_start >> 6; const uint32_t end = pos_end >> 6; const uint64_t first = ~((1ULL << (pos_start & 0x3F)) - 1); const uint64_t last = (1ULL << (pos_end & 0x3F)) - 1; - if (start == end) return ((bitset->words[end] & first & last) == (first & last)); + if (start == end) + return ((bitset->words[end] & first & last) == (first & last)); if ((bitset->words[start] & first) != first) return false; - if ((end < BITSET_CONTAINER_SIZE_IN_WORDS) && ((bitset->words[end] & last) != last)){ - + if ((end < BITSET_CONTAINER_SIZE_IN_WORDS) && + ((bitset->words[end] & last) != last)) { return false; } - for (uint16_t i = start + 1; (i < BITSET_CONTAINER_SIZE_IN_WORDS) && (i < end); ++i){ - + for (uint32_t i = start + 1; + (i < BITSET_CONTAINER_SIZE_IN_WORDS) && (i < end); ++i) { if (bitset->words[i] != UINT64_C(0xFFFFFFFFFFFFFFFF)) return false; } @@ -2025,11 +2074,11 @@ inline bool bitset_container_contains(const bitset_container_t *bitset, } /* -* Check whether a range of bits from position `pos_start' (included) to `pos_end' (excluded) -* is present in `bitset'. Calls bitset_container_get_all. -*/ -static inline bool bitset_container_contains_range(const bitset_container_t *bitset, - uint32_t pos_start, uint32_t pos_end) { + * Check whether a range of bits from position `pos_start' (included) to + * `pos_end' (excluded) is present in `bitset'. Calls bitset_container_get_all. + */ +static inline bool bitset_container_contains_range( + const bitset_container_t *bitset, uint32_t pos_start, uint32_t pos_end) { return bitset_container_get_range(bitset, pos_start, pos_end); } @@ -2040,9 +2089,6 @@ static inline int bitset_container_cardinality( return bitset->cardinality; } - - - /* Copy one container into another. We assume that they are distinct. */ void bitset_container_copy(const bitset_container_t *source, bitset_container_t *dest); @@ -2059,20 +2105,18 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset); /* Check whether this bitset is empty, * it never modifies the bitset struct. */ -static inline bool bitset_container_empty( - const bitset_container_t *bitset) { - if (bitset->cardinality == BITSET_UNKNOWN_CARDINALITY) { - for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i ++) { - if((bitset->words[i]) != 0) return false; - } - return true; - } - return bitset->cardinality == 0; +static inline bool bitset_container_empty(const bitset_container_t *bitset) { + if (bitset->cardinality == BITSET_UNKNOWN_CARDINALITY) { + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { + if ((bitset->words[i]) != 0) return false; + } + return true; + } + return bitset->cardinality == 0; } - -/* Get whether there is at least one bit set (see bitset_container_empty for the reverse), - the bitset is never modified */ +/* Get whether there is at least one bit set (see bitset_container_empty for + the reverse), the bitset is never modified */ static inline bool bitset_container_const_nonzero_cardinality( const bitset_container_t *bitset) { return !bitset_container_empty(bitset); @@ -2082,7 +2126,7 @@ static inline bool bitset_container_const_nonzero_cardinality( * Check whether the two bitsets intersect */ bool bitset_container_intersect(const bitset_container_t *src_1, - const bitset_container_t *src_2); + const bitset_container_t *src_2); /* Computes the union of bitsets `src_1' and `src_2' into `dst' and return the * cardinality. */ @@ -2109,8 +2153,8 @@ int bitset_container_union_justcard(const bitset_container_t *src_1, /* Computes the union of bitsets `src_1' and `src_2' into `dst', but does * not update the cardinality. Provided to optimize chained operations. */ int bitset_container_union_nocard(const bitset_container_t *src_1, - const bitset_container_t *src_2, - bitset_container_t *dst); + const bitset_container_t *src_2, + bitset_container_t *dst); /* Computes the union of bitsets `src_1' and `src_2' into `dst', but does not * update the cardinality. Provided to optimize chained operations. */ @@ -2143,8 +2187,8 @@ int bitset_container_intersection_justcard(const bitset_container_t *src_1, /* Computes the intersection of bitsets `src_1' and `src_2' into `dst', but does * not update the cardinality. Provided to optimize chained operations. */ int bitset_container_intersection_nocard(const bitset_container_t *src_1, - const bitset_container_t *src_2, - bitset_container_t *dst); + const bitset_container_t *src_2, + bitset_container_t *dst); /* Computes the intersection of bitsets `src_1' and `src_2' into `dst', but does * not update the cardinality. Provided to optimize chained operations. */ @@ -2186,9 +2230,8 @@ int bitset_container_andnot_nocard(const bitset_container_t *src_1, const bitset_container_t *src_2, bitset_container_t *dst); -void bitset_container_offset(const bitset_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset); +void bitset_container_offset(const bitset_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); /* * Write out the 16-bit integers contained in this container as a list of 32-bit * integers using base @@ -2215,7 +2258,8 @@ void bitset_container_printf(const bitset_container_t *v); void bitset_container_printf_as_uint32_array(const bitset_container_t *v, uint32_t base); -bool bitset_container_validate(const bitset_container_t *v, const char **reason); +bool bitset_container_validate(const bitset_container_t *v, + const char **reason); /** * Return the serialized size in bytes of a container. @@ -2273,8 +2317,8 @@ bool bitset_container_equals(const bitset_container_t *container1, const bitset_container_t *container2); /** -* Return true if container1 is a subset of container2. -*/ + * Return true if container1 is a subset of container2. + */ bool bitset_container_is_subset(const bitset_container_t *container1, const bitset_container_t *container2); @@ -2297,14 +2341,23 @@ uint16_t bitset_container_maximum(const bitset_container_t *container); /* Returns the number of values equal or smaller than x */ int bitset_container_rank(const bitset_container_t *container, uint16_t x); +/* bulk version of bitset_container_rank(); return number of consumed elements + */ +uint32_t bitset_container_rank_many(const bitset_container_t *container, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + /* Returns the index of x , if not exsist return -1 */ int bitset_container_get_index(const bitset_container_t *container, uint16_t x); /* Returns the index of the first value equal or larger than x, or -1 */ -int bitset_container_index_equalorlarger(const bitset_container_t *container, uint16_t x); +int bitset_container_index_equalorlarger(const bitset_container_t *container, + uint16_t x); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_BITSET_H_ */ @@ -2318,15 +2371,17 @@ int bitset_container_index_equalorlarger(const bitset_container_t *container, ui #ifndef INCLUDE_CONTAINERS_RUN_H_ #define INCLUDE_CONTAINERS_RUN_H_ + +// Include other headers after roaring_types.h #include #include #include #include - #ifdef __cplusplus -extern "C" { namespace roaring { +extern "C" { +namespace roaring { // Note: in pure C++ code, you should avoid putting `using` in header files using api::roaring_iterator; @@ -2351,11 +2406,11 @@ struct rle16_s { typedef struct rle16_s rle16_t; #ifdef __cplusplus - #define MAKE_RLE16(val,len) \ - {(uint16_t)(val), (uint16_t)(len)} // no tagged structs until c++20 +#define CROARING_MAKE_RLE16(val, len) \ + { (uint16_t)(val), (uint16_t)(len) } // no tagged structs until c++20 #else - #define MAKE_RLE16(val,len) \ - (rle16_t){.value = (uint16_t)(val), .length = (uint16_t)(len)} +#define CROARING_MAKE_RLE16(val, len) \ + (rle16_t) { .value = (uint16_t)(val), .length = (uint16_t)(len) } #endif /* struct run_container_s - run container bitmap @@ -2372,8 +2427,8 @@ STRUCT_CONTAINER(run_container_s) { typedef struct run_container_s run_container_t; -#define CAST_run(c) CAST(run_container_t *, c) // safer downcast -#define const_CAST_run(c) CAST(const run_container_t *, c) +#define CAST_run(c) CAST(run_container_t *, c) // safer downcast +#define const_CAST_run(c) CAST(const run_container_t *, c) #define movable_CAST_run(c) movable_CAST(run_container_t **, c) /* Create a new run container. Return NULL in case of failure. */ @@ -2446,13 +2501,12 @@ static inline int32_t rle16_find_run(const rle16_t *array, int32_t lenarray, return -(low + 1); } - /** * Returns number of runs which can'be be merged with the key because they * are less than the key. * Note that [5,6,7,8] can be merged with the key 9 and won't be counted. */ -static inline int32_t rle16_count_less(const rle16_t* array, int32_t lenarray, +static inline int32_t rle16_count_less(const rle16_t *array, int32_t lenarray, uint16_t key) { if (lenarray == 0) return 0; int32_t low = 0; @@ -2460,8 +2514,9 @@ static inline int32_t rle16_count_less(const rle16_t* array, int32_t lenarray, while (low <= high) { int32_t middleIndex = (low + high) >> 1; uint16_t min_value = array[middleIndex].value; - uint16_t max_value = array[middleIndex].value + array[middleIndex].length; - if (max_value + UINT32_C(1) < key) { // uint32 arithmetic + uint16_t max_value = + array[middleIndex].value + array[middleIndex].length; + if (max_value + UINT32_C(1) < key) { // uint32 arithmetic low = middleIndex + 1; } else if (key < min_value) { high = middleIndex - 1; @@ -2472,18 +2527,19 @@ static inline int32_t rle16_count_less(const rle16_t* array, int32_t lenarray, return low; } -static inline int32_t rle16_count_greater(const rle16_t* array, int32_t lenarray, - uint16_t key) { +static inline int32_t rle16_count_greater(const rle16_t *array, + int32_t lenarray, uint16_t key) { if (lenarray == 0) return 0; int32_t low = 0; int32_t high = lenarray - 1; while (low <= high) { int32_t middleIndex = (low + high) >> 1; uint16_t min_value = array[middleIndex].value; - uint16_t max_value = array[middleIndex].value + array[middleIndex].length; + uint16_t max_value = + array[middleIndex].value + array[middleIndex].length; if (max_value < key) { low = middleIndex + 1; - } else if (key + UINT32_C(1) < min_value) { // uint32 arithmetic + } else if (key + UINT32_C(1) < min_value) { // uint32 arithmetic high = middleIndex - 1; } else { return lenarray - (middleIndex + 1); @@ -2567,16 +2623,19 @@ inline bool run_container_contains(const run_container_t *run, uint16_t pos) { } /* -* Check whether all positions in a range of positions from pos_start (included) -* to pos_end (excluded) is present in `run'. -*/ + * Check whether all positions in a range of positions from pos_start (included) + * to pos_end (excluded) is present in `run'. + */ static inline bool run_container_contains_range(const run_container_t *run, - uint32_t pos_start, uint32_t pos_end) { + uint32_t pos_start, + uint32_t pos_end) { uint32_t count = 0; - int32_t index = interleavedBinarySearch(run->runs, run->n_runs, pos_start); + int32_t index = + interleavedBinarySearch(run->runs, run->n_runs, (uint16_t)pos_start); if (index < 0) { index = -index - 2; - if ((index == -1) || ((pos_start - run->runs[index].value) > run->runs[index].length)){ + if ((index == -1) || + ((pos_start - run->runs[index].value) > run->runs[index].length)) { return false; } } @@ -2584,7 +2643,9 @@ static inline bool run_container_contains_range(const run_container_t *run, const uint32_t stop = run->runs[i].value + run->runs[i].length; if (run->runs[i].value >= pos_end) break; if (stop >= pos_end) { - count += (((pos_end - run->runs[i].value) > 0) ? (pos_end - run->runs[i].value) : 0); + count += (((pos_end - run->runs[i].value) > 0) + ? (pos_end - run->runs[i].value) + : 0); break; } const uint32_t min = (stop - pos_start) > 0 ? (stop - pos_start) : 0; @@ -2603,13 +2664,10 @@ static inline bool run_container_nonzero_cardinality( } /* Card == 0?, see run_container_nonzero_cardinality for the reverse */ -static inline bool run_container_empty( - const run_container_t *run) { +static inline bool run_container_empty(const run_container_t *run) { return run->n_runs == 0; // runs never empty } - - /* Copy one container into another. We assume that they are distinct. */ void run_container_copy(const run_container_t *src, run_container_t *dst); @@ -2663,7 +2721,7 @@ static inline void run_container_append_value(run_container_t *run, rle16_t *previousrl) { const uint32_t previousend = previousrl->value + previousrl->length; if (val > previousend + 1) { // we add a new one - *previousrl = MAKE_RLE16(val, 0); + *previousrl = CROARING_MAKE_RLE16(val, 0); run->runs[run->n_runs] = *previousrl; run->n_runs++; } else if (val == previousend + 1) { // we merge @@ -2678,7 +2736,7 @@ static inline void run_container_append_value(run_container_t *run, */ static inline rle16_t run_container_append_value_first(run_container_t *run, uint16_t val) { - rle16_t newrle = MAKE_RLE16(val, 0); + rle16_t newrle = CROARING_MAKE_RLE16(val, 0); run->runs[run->n_runs] = newrle; run->n_runs++; return newrle; @@ -2712,7 +2770,7 @@ int run_container_intersection_cardinality(const run_container_t *src_1, /* Check whether src_1 and src_2 intersect. */ bool run_container_intersect(const run_container_t *src_1, - const run_container_t *src_2); + const run_container_t *src_2); /* Compute the symmetric difference of `src_1' and `src_2' and write the result * to `dst' @@ -2783,6 +2841,7 @@ int32_t run_container_read(int32_t cardinality, run_container_t *container, * Return the serialized size in bytes of a container (see run_container_write). * This is meant to be compatible with the Java and Go versions of Roaring. */ +ALLOW_UNALIGNED static inline int32_t run_container_size_in_bytes( const run_container_t *container) { return run_container_serialized_size_in_bytes(container->n_runs); @@ -2793,7 +2852,7 @@ static inline int32_t run_container_size_in_bytes( */ ALLOW_UNALIGNED static inline bool run_container_equals(const run_container_t *container1, - const run_container_t *container2) { + const run_container_t *container2) { if (container1->n_runs != container2->n_runs) { return false; } @@ -2802,8 +2861,8 @@ static inline bool run_container_equals(const run_container_t *container1, } /** -* Return true if container1 is a subset of container2. -*/ + * Return true if container1 is a subset of container2. + */ bool run_container_is_subset(const run_container_t *container1, const run_container_t *container2); @@ -2816,12 +2875,12 @@ void run_container_smart_append_exclusive(run_container_t *src, const uint16_t length); /** -* The new container consists of a single run [start,stop). -* It is required that stop>start, the caller is responsability for this check. -* It is required that stop <= (1<<16), the caller is responsability for this check. -* The cardinality of the created container is stop - start. -* Returns NULL on failure -*/ + * The new container consists of a single run [start,stop). + * It is required that stop>start, the caller is responsability for this check. + * It is required that stop <= (1<<16), the caller is responsability for this + * check. The cardinality of the created container is stop - start. Returns NULL + * on failure + */ static inline run_container_t *run_container_create_range(uint32_t start, uint32_t stop) { run_container_t *rc = run_container_create_given_capacity(1); @@ -2850,9 +2909,8 @@ bool run_container_select(const run_container_t *container, void run_container_andnot(const run_container_t *src_1, const run_container_t *src_2, run_container_t *dst); -void run_container_offset(const run_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset); +void run_container_offset(const run_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); /* Returns the smallest value (assumes not empty) */ inline uint16_t run_container_minimum(const run_container_t *run) { @@ -2869,11 +2927,18 @@ inline uint16_t run_container_maximum(const run_container_t *run) { /* Returns the number of values equal or smaller than x */ int run_container_rank(const run_container_t *arr, uint16_t x); +/* bulk version of run_container_rank(); return number of consumed elements */ +uint32_t run_container_rank_many(const run_container_t *arr, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + /* Returns the index of x, if not exsist return -1 */ int run_container_get_index(const run_container_t *arr, uint16_t x); -/* Returns the index of the first run containing a value at least as large as x, or -1 */ -inline int run_container_index_equalorlarger(const run_container_t *arr, uint16_t x) { +/* Returns the index of the first run containing a value at least as large as x, + * or -1 */ +inline int run_container_index_equalorlarger(const run_container_t *arr, + uint16_t x) { int32_t index = interleavedBinarySearch(arr->runs, arr->n_runs, x); if (index >= 0) return index; index = -index - 2; // points to preceding run, possibly -1 @@ -2883,8 +2948,8 @@ inline int run_container_index_equalorlarger(const run_container_t *arr, uint16_ if (offset <= le) return index; } index += 1; - if(index < arr->n_runs) { - return index; + if (index < arr->n_runs) { + return index; } return -1; } @@ -2892,15 +2957,15 @@ inline int run_container_index_equalorlarger(const run_container_t *arr, uint16_ /* * Add all values in range [min, max] using hint. */ -static inline void run_container_add_range_nruns(run_container_t* run, +static inline void run_container_add_range_nruns(run_container_t *run, uint32_t min, uint32_t max, int32_t nruns_less, int32_t nruns_greater) { int32_t nruns_common = run->n_runs - nruns_less - nruns_greater; if (nruns_common == 0) { - makeRoomAtIndex(run, nruns_less); - run->runs[nruns_less].value = min; - run->runs[nruns_less].length = max - min; + makeRoomAtIndex(run, (uint16_t)nruns_less); + run->runs[nruns_less].value = (uint16_t)min; + run->runs[nruns_less].length = (uint16_t)(max - min); } else { uint32_t common_min = run->runs[nruns_less].value; uint32_t common_max = run->runs[nruns_less + nruns_common - 1].value + @@ -2908,12 +2973,12 @@ static inline void run_container_add_range_nruns(run_container_t* run, uint32_t result_min = (common_min < min) ? common_min : min; uint32_t result_max = (common_max > max) ? common_max : max; - run->runs[nruns_less].value = result_min; - run->runs[nruns_less].length = result_max - result_min; + run->runs[nruns_less].value = (uint16_t)result_min; + run->runs[nruns_less].length = (uint16_t)(result_max - result_min); memmove(&(run->runs[nruns_less + 1]), &(run->runs[run->n_runs - nruns_greater]), - nruns_greater*sizeof(rle16_t)); + nruns_greater * sizeof(rle16_t)); run->n_runs = nruns_less + 1 + nruns_greater; } } @@ -2925,44 +2990,52 @@ static inline void run_container_add_range_nruns(run_container_t* run, /*static inline void run_container_add_range(run_container_t* run, uint32_t min, uint32_t max) { int32_t nruns_greater = rle16_count_greater(run->runs, run->n_runs, max); - int32_t nruns_less = rle16_count_less(run->runs, run->n_runs - nruns_greater, min); - run_container_add_range_nruns(run, min, max, nruns_less, nruns_greater); + int32_t nruns_less = rle16_count_less(run->runs, run->n_runs - +nruns_greater, min); run_container_add_range_nruns(run, min, max, nruns_less, +nruns_greater); }*/ /** - * Shifts last $count elements either left (distance < 0) or right (distance > 0) + * Shifts last $count elements either left (distance < 0) or right (distance > + * 0) */ -static inline void run_container_shift_tail(run_container_t* run, - int32_t count, int32_t distance) { +static inline void run_container_shift_tail(run_container_t *run, int32_t count, + int32_t distance) { if (distance > 0) { - if (run->capacity < count+distance) { - run_container_grow(run, count+distance, true); + if (run->capacity < count + distance) { + run_container_grow(run, count + distance, true); } } int32_t srcpos = run->n_runs - count; int32_t dstpos = srcpos + distance; - memmove(&(run->runs[dstpos]), &(run->runs[srcpos]), sizeof(rle16_t) * count); + memmove(&(run->runs[dstpos]), &(run->runs[srcpos]), + sizeof(rle16_t) * count); run->n_runs += distance; } /** * Remove all elements in range [min, max] */ -static inline void run_container_remove_range(run_container_t *run, uint32_t min, uint32_t max) { - int32_t first = rle16_find_run(run->runs, run->n_runs, min); - int32_t last = rle16_find_run(run->runs, run->n_runs, max); +static inline void run_container_remove_range(run_container_t *run, + uint32_t min, uint32_t max) { + int32_t first = rle16_find_run(run->runs, run->n_runs, (uint16_t)min); + int32_t last = rle16_find_run(run->runs, run->n_runs, (uint16_t)max); if (first >= 0 && min > run->runs[first].value && - max < ((uint32_t)run->runs[first].value + (uint32_t)run->runs[first].length)) { + max < ((uint32_t)run->runs[first].value + + (uint32_t)run->runs[first].length)) { // split this run into two adjacent runs // right subinterval - makeRoomAtIndex(run, first+1); - run->runs[first+1].value = max + 1; - run->runs[first+1].length = (run->runs[first].value + run->runs[first].length) - (max + 1); + makeRoomAtIndex(run, (uint16_t)(first + 1)); + run->runs[first + 1].value = (uint16_t)(max + 1); + run->runs[first + 1].length = + (uint16_t)((run->runs[first].value + run->runs[first].length) - + (max + 1)); // left subinterval - run->runs[first].length = (min - 1) - run->runs[first].value; + run->runs[first].length = + (uint16_t)((min - 1) - run->runs[first].value); return; } @@ -2970,33 +3043,37 @@ static inline void run_container_remove_range(run_container_t *run, uint32_t min // update left-most partial run if (first >= 0) { if (min > run->runs[first].value) { - run->runs[first].length = (min - 1) - run->runs[first].value; + run->runs[first].length = + (uint16_t)((min - 1) - run->runs[first].value); first++; } } else { - first = -first-1; + first = -first - 1; } // update right-most run if (last >= 0) { uint16_t run_max = run->runs[last].value + run->runs[last].length; if (run_max > max) { - run->runs[last].value = max + 1; - run->runs[last].length = run_max - (max + 1); + run->runs[last].value = (uint16_t)(max + 1); + run->runs[last].length = (uint16_t)(run_max - (max + 1)); last--; } } else { - last = (-last-1) - 1; + last = (-last - 1) - 1; } // remove intermediate runs if (first <= last) { - run_container_shift_tail(run, run->n_runs - (last+1), -(last-first+1)); + run_container_shift_tail(run, run->n_runs - (last + 1), + -(last - first + 1)); } } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_RUN_H_ */ @@ -3012,7 +3089,9 @@ static inline void run_container_remove_range(run_container_t *run, uint32_t min #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Convert an array into a bitset. The input container is not freed or modified. @@ -3035,39 +3114,38 @@ run_container_t *run_container_from_array(const array_container_t *c); /* convert a run into either an array or a bitset * might free the container. This does not free the input run container. */ -container_t *convert_to_bitset_or_array_container( - run_container_t *rc, int32_t card, - uint8_t *resulttype); +container_t *convert_to_bitset_or_array_container(run_container_t *rc, + int32_t card, + uint8_t *resulttype); /* convert containers to and from runcontainers, as is most space efficient. * The container might be freed. */ -container_t *convert_run_optimize( - container_t *c, uint8_t typecode_original, - uint8_t *typecode_after); +container_t *convert_run_optimize(container_t *c, uint8_t typecode_original, + uint8_t *typecode_after); /* converts a run container to either an array or a bitset, IF it saves space. */ /* If a conversion occurs, the caller is responsible to free the original * container and * he becomes reponsible to free the new one. */ -container_t *convert_run_to_efficient_container( - run_container_t *c, uint8_t *typecode_after); +container_t *convert_run_to_efficient_container(run_container_t *c, + uint8_t *typecode_after); // like convert_run_to_efficient_container but frees the old result if needed container_t *convert_run_to_efficient_container_and_free( - run_container_t *c, uint8_t *typecode_after); + run_container_t *c, uint8_t *typecode_after); /** * Create new container which is a union of run container and * range [min, max]. Caller is responsible for freeing run container. */ -container_t *container_from_run_range( - const run_container_t *run, - uint32_t min, uint32_t max, - uint8_t *typecode_after); +container_t *container_from_run_range(const run_container_t *run, uint32_t min, + uint32_t max, uint8_t *typecode_after); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_CONVERT_H_ */ @@ -3083,7 +3161,9 @@ container_t *container_from_run_range( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /** @@ -3104,7 +3184,9 @@ bool run_container_equals_bitset(const run_container_t* container1, const bitset_container_t* container2); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* CONTAINERS_MIXED_EQUAL_H_ */ @@ -3120,7 +3202,9 @@ bool run_container_equals_bitset(const run_container_t* container1, #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /** @@ -3130,31 +3214,33 @@ bool array_container_is_subset_bitset(const array_container_t* container1, const bitset_container_t* container2); /** -* Return true if container1 is a subset of container2. + * Return true if container1 is a subset of container2. */ bool run_container_is_subset_array(const run_container_t* container1, const array_container_t* container2); /** -* Return true if container1 is a subset of container2. + * Return true if container1 is a subset of container2. */ bool array_container_is_subset_run(const array_container_t* container1, const run_container_t* container2); /** -* Return true if container1 is a subset of container2. + * Return true if container1 is a subset of container2. */ bool run_container_is_subset_bitset(const run_container_t* container1, const bitset_container_t* container2); /** -* Return true if container1 is a subset of container2. -*/ + * Return true if container1 is a subset of container2. + */ bool bitset_container_is_subset_run(const bitset_container_t* container1, const run_container_t* container2); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* CONTAINERS_MIXED_SUBSET_H_ */ @@ -3168,7 +3254,9 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the andnot of src_1 and src_2 and write the result to @@ -3188,9 +3276,9 @@ void array_bitset_container_iandnot(array_container_t *src_1, * Return true for a bitset result; false for array */ -bool bitset_array_container_andnot( - const bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool bitset_array_container_andnot(const bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst (which has no container initially). It will modify src_1 @@ -3199,9 +3287,9 @@ bool bitset_array_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_array_container_iandnot( - bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool bitset_array_container_iandnot(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst. Result may be either a bitset or an array container @@ -3210,9 +3298,9 @@ bool bitset_array_container_iandnot( * result true) or an array container. */ -bool run_bitset_container_andnot( - const run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool run_bitset_container_andnot(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst. Result may be either a bitset or an array container @@ -3221,9 +3309,9 @@ bool run_bitset_container_andnot( * result true) or an array container. */ -bool run_bitset_container_iandnot( - run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool run_bitset_container_iandnot(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst. Result may be either a bitset or an array container @@ -3232,9 +3320,9 @@ bool run_bitset_container_iandnot( * result true) or an array container. */ -bool bitset_run_container_andnot( - const bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst); +bool bitset_run_container_andnot(const bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst (which has no container initially). It will modify src_1 @@ -3243,17 +3331,17 @@ bool bitset_run_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_run_container_iandnot( - bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst); +bool bitset_run_container_iandnot(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst); /* dst does not indicate a valid container initially. Eventually it * can become any type of container. */ -int run_array_container_andnot( - const run_container_t *src_1, const array_container_t *src_2, - container_t **dst); +int run_array_container_andnot(const run_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst (which has no container initially). It will modify src_1 @@ -3262,9 +3350,9 @@ int run_array_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -int run_array_container_iandnot( - run_container_t *src_1, const array_container_t *src_2, - container_t **dst); +int run_array_container_iandnot(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* dst must be a valid array container, allowed to be src_1 */ @@ -3283,9 +3371,8 @@ void array_run_container_iandnot(array_container_t *src_1, * can become any kind of container. */ -int run_run_container_andnot( - const run_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int run_run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst (which has no container initially). It will modify src_1 @@ -3294,9 +3381,8 @@ int run_run_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -int run_run_container_iandnot( - run_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int run_run_container_iandnot(run_container_t *src_1, + const run_container_t *src_2, container_t **dst); /* * dst is a valid array container and may be the same as src_1 @@ -3316,9 +3402,9 @@ void array_array_container_iandnot(array_container_t *src_1, * "dst is a bitset" */ -bool bitset_bitset_container_andnot( - const bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool bitset_bitset_container_andnot(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the andnot of src_1 and src_2 and write the result to * dst (which has no container initially). It will modify src_1 @@ -3327,12 +3413,14 @@ bool bitset_bitset_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_bitset_container_iandnot( - bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool bitset_bitset_container_iandnot(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif @@ -3353,7 +3441,9 @@ bool bitset_bitset_container_iandnot( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the intersection of src_1 and src_2 and write the result to @@ -3367,11 +3457,9 @@ void array_bitset_container_intersection(const array_container_t *src_1, int array_bitset_container_intersection_cardinality( const array_container_t *src_1, const bitset_container_t *src_2); - - /* Checking whether src_1 and src_2 intersect. */ bool array_bitset_container_intersect(const array_container_t *src_1, - const bitset_container_t *src_2); + const bitset_container_t *src_2); /* * Compute the intersection between src_1 and src_2 and write the result @@ -3405,18 +3493,17 @@ int array_run_container_intersection_cardinality(const array_container_t *src_1, /* Compute the size of the intersection between src_1 and src_2 **/ -int run_bitset_container_intersection_cardinality(const run_container_t *src_1, - const bitset_container_t *src_2); - +int run_bitset_container_intersection_cardinality( + const run_container_t *src_1, const bitset_container_t *src_2); /* Check that src_1 and src_2 intersect. */ bool array_run_container_intersect(const array_container_t *src_1, - const run_container_t *src_2); + const run_container_t *src_2); /* Check that src_1 and src_2 intersect. **/ bool run_bitset_container_intersect(const run_container_t *src_1, - const bitset_container_t *src_2); + const bitset_container_t *src_2); /* * Same as bitset_bitset_container_intersection except that if the output is to @@ -3431,7 +3518,9 @@ bool bitset_bitset_container_intersection_inplace( container_t **dst); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ */ @@ -3447,7 +3536,9 @@ bool bitset_bitset_container_intersection_inplace( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Negation across the entire range of the container. @@ -3467,9 +3558,8 @@ void array_container_negation(const array_container_t *src, * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -bool bitset_container_negation( - const bitset_container_t *src, - container_t **dst); +bool bitset_container_negation(const bitset_container_t *src, + container_t **dst); /* inplace version */ /* @@ -3480,9 +3570,8 @@ bool bitset_container_negation( * to free the container. * In all cases, the result is in *dst. */ -bool bitset_container_negation_inplace( - bitset_container_t *src, - container_t **dst); +bool bitset_container_negation_inplace(bitset_container_t *src, + container_t **dst); /* Negation across the entire range of container * Compute the negation of src and write the result @@ -3507,19 +3596,18 @@ int run_container_negation_inplace(run_container_t *src, container_t **dst); * to *dst. Returns true if the result is a bitset container * and false for an array container. *dst is not preallocated. */ -bool array_container_negation_range( - const array_container_t *src, - const int range_start, const int range_end, - container_t **dst); +bool array_container_negation_range(const array_container_t *src, + const int range_start, const int range_end, + container_t **dst); /* Even when the result would fit, it is unclear how to make an * inplace version without inefficient copying. Thus this routine * may be a wrapper for the non-in-place version */ -bool array_container_negation_range_inplace( - array_container_t *src, - const int range_start, const int range_end, - container_t **dst); +bool array_container_negation_range_inplace(array_container_t *src, + const int range_start, + const int range_end, + container_t **dst); /* Negation across a range of the container * Compute the negation of src and write the result @@ -3528,10 +3616,9 @@ bool array_container_negation_range_inplace( * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -bool bitset_container_negation_range( - const bitset_container_t *src, - const int range_start, const int range_end, - container_t **dst); +bool bitset_container_negation_range(const bitset_container_t *src, + const int range_start, const int range_end, + container_t **dst); /* inplace version */ /* @@ -3542,10 +3629,10 @@ bool bitset_container_negation_range( * to free the container. * In all cases, the result is in *dst. */ -bool bitset_container_negation_range_inplace( - bitset_container_t *src, - const int range_start, const int range_end, - container_t **dst); +bool bitset_container_negation_range_inplace(bitset_container_t *src, + const int range_start, + const int range_end, + container_t **dst); /* Negation across a range of container * Compute the negation of src and write the result @@ -3553,10 +3640,9 @@ bool bitset_container_negation_range_inplace( * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -int run_container_negation_range( - const run_container_t *src, - const int range_start, const int range_end, - container_t **dst); +int run_container_negation_range(const run_container_t *src, + const int range_start, const int range_end, + container_t **dst); /* * Same as run_container_negation except that if the output is to @@ -3565,13 +3651,15 @@ int run_container_negation_range( * then src is modified and no allocation is made. * In all cases, the result is in *dst. */ -int run_container_negation_range_inplace( - run_container_t *src, - const int range_start, const int range_end, - container_t **dst); +int run_container_negation_range_inplace(run_container_t *src, + const int range_start, + const int range_end, + container_t **dst); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_MIXED_NEGATION_H_ */ @@ -3592,7 +3680,9 @@ int run_container_negation_range_inplace( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the union of src_1 and src_2 and write the result to @@ -3614,9 +3704,9 @@ void array_bitset_container_lazy_union(const array_container_t *src_1, * otherwise is a array_container_t. We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -bool array_array_container_union( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* * Compute the union between src_1 and src_2 and write the result @@ -3624,27 +3714,28 @@ bool array_array_container_union( * the result is a bitset_container_t * otherwise is a array_container_t. When the result is an array_container_t, it * it either written to src_1 (if *dst is null) or to *dst. - * If the result is a bitset_container_t and *dst is null, then there was a failure. + * If the result is a bitset_container_t and *dst is null, then there was a + * failure. */ -bool array_array_container_inplace_union( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* * Same as array_array_container_union except that it will more eagerly produce * a bitset. */ -bool array_array_container_lazy_union( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_lazy_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* - * Same as array_array_container_inplace_union except that it will more eagerly produce - * a bitset. + * Same as array_array_container_inplace_union except that it will more eagerly + * produce a bitset. */ -bool array_array_container_lazy_inplace_union( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_lazy_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* Compute the union of src_1 and src_2 and write the result to * dst. We assume that dst is a @@ -3682,7 +3773,9 @@ void run_bitset_container_lazy_union(const run_container_t *src_1, bitset_container_t *dst); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif /* INCLUDE_CONTAINERS_MIXED_UNION_H_ */ @@ -3710,15 +3803,17 @@ void run_bitset_container_lazy_union(const run_container_t *src_1, #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the xor of src_1 and src_2 and write the result to * dst (which has no container initially). * Result is true iff dst is a bitset */ -bool array_bitset_container_xor( - const array_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool array_bitset_container_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the xor of src_1 and src_2 and write the result to * dst. It is allowed for src_2 to be dst. This version does not @@ -3733,9 +3828,9 @@ void array_bitset_container_lazy_xor(const array_container_t *src_1, * "dst is a bitset" */ -bool bitset_bitset_container_xor( - const bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool bitset_bitset_container_xor(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the xor of src_1 and src_2 and write the result to * dst. Result may be either a bitset or an array container @@ -3744,9 +3839,9 @@ bool bitset_bitset_container_xor( * result true) or an array container. */ -bool run_bitset_container_xor( - const run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool run_bitset_container_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* lazy xor. Dst is initialized and may be equal to src_2. * Result is left as a bitset container, even if actual @@ -3761,17 +3856,16 @@ void run_bitset_container_lazy_xor(const run_container_t *src_1, * can become any kind of container. */ -int array_run_container_xor( - const array_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int array_run_container_xor(const array_container_t *src_1, + const run_container_t *src_2, container_t **dst); /* dst does not initially have a valid container. Creates either * an array or a bitset container, indicated by return code */ -bool array_array_container_xor( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* dst does not initially have a valid container. Creates either * an array or a bitset container, indicated by return code. @@ -3779,9 +3873,9 @@ bool array_array_container_xor( * container type might not be correct for the actual cardinality */ -bool array_array_container_lazy_xor( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_lazy_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); /* Dst is a valid run container. (Can it be src_2? Let's say not.) * Leaves result as run container, even if other options are @@ -3796,9 +3890,8 @@ void array_run_container_lazy_xor(const array_container_t *src_1, * can become any kind of container. */ -int run_run_container_xor( - const run_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int run_run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst); /* INPLACE versions (initial implementation may not exploit all inplace * opportunities (if any...) @@ -3811,17 +3904,17 @@ int run_run_container_xor( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_array_container_ixor( - bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool bitset_array_container_ixor(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); -bool bitset_bitset_container_ixor( - bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool bitset_bitset_container_ixor(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); -bool array_bitset_container_ixor( - array_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool array_bitset_container_ixor(array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); /* Compute the xor of src_1 and src_2 and write the result to * dst. Result may be either a bitset or an array container @@ -3830,36 +3923,34 @@ bool array_bitset_container_ixor( * result true) or an array container. */ -bool run_bitset_container_ixor( - run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst); +bool run_bitset_container_ixor(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); -bool bitset_run_container_ixor( - bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst); +bool bitset_run_container_ixor(bitset_container_t *src_1, + const run_container_t *src_2, container_t **dst); /* dst does not indicate a valid container initially. Eventually it * can become any kind of container. */ -int array_run_container_ixor( - array_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int array_run_container_ixor(array_container_t *src_1, + const run_container_t *src_2, container_t **dst); -int run_array_container_ixor( - run_container_t *src_1, const array_container_t *src_2, - container_t **dst); +int run_array_container_ixor(run_container_t *src_1, + const array_container_t *src_2, container_t **dst); -bool array_array_container_ixor( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst); +bool array_array_container_ixor(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); -int run_run_container_ixor( - run_container_t *src_1, const run_container_t *src_2, - container_t **dst); +int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, + container_t **dst); #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif @@ -3874,7 +3965,9 @@ int run_run_container_ixor( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif // would enum be possible or better? @@ -3900,10 +3993,9 @@ extern "C" { namespace roaring { namespace internal { * ... * } */ -#define PAIR_CONTAINER_TYPES(type1,type2) \ - (4 * (type1) + (type2)) +#define PAIR_CONTAINER_TYPES(type1, type2) (4 * (type1) + (type2)) -#define CONTAINER_PAIR(name1,name2) \ +#define CONTAINER_PAIR(name1, name2) \ (4 * (name1##_CONTAINER_TYPE) + (name2##_CONTAINER_TYPE)) /** @@ -3918,8 +4010,8 @@ STRUCT_CONTAINER(shared_container_s) { typedef struct shared_container_s shared_container_t; -#define CAST_shared(c) CAST(shared_container_t *, c) // safer downcast -#define const_CAST_shared(c) CAST(const shared_container_t *, c) +#define CAST_shared(c) CAST(shared_container_t *, c) // safer downcast +#define const_CAST_shared(c) CAST(const shared_container_t *, c) #define movable_CAST_shared(c) movable_CAST(shared_container_t **, c) /* @@ -3945,8 +4037,7 @@ container_t *shared_container_extract_copy(shared_container_t *container, /* access to container underneath */ static inline const container_t *container_unwrap_shared( - const container_t *candidate_shared_container, uint8_t *type -){ + const container_t *candidate_shared_container, uint8_t *type) { if (*type == SHARED_CONTAINER_TYPE) { *type = const_CAST_shared(candidate_shared_container)->typecode; assert(*type != SHARED_CONTAINER_TYPE); @@ -3956,11 +4047,9 @@ static inline const container_t *container_unwrap_shared( } } - /* access to container underneath */ -static inline container_t *container_mutable_unwrap_shared( - container_t *c, uint8_t *type -) { +static inline container_t *container_mutable_unwrap_shared(container_t *c, + uint8_t *type) { if (*type == SHARED_CONTAINER_TYPE) { // the passed in container is shared *type = CAST_shared(c)->typecode; assert(*type != SHARED_CONTAINER_TYPE); @@ -3971,9 +4060,7 @@ static inline container_t *container_mutable_unwrap_shared( } /* access to container underneath and queries its type */ -static inline uint8_t get_container_type( - const container_t *c, uint8_t type -){ +static inline uint8_t get_container_type(const container_t *c, uint8_t type) { if (type == SHARED_CONTAINER_TYPE) { return const_CAST_shared(c)->typecode; } else { @@ -3989,9 +4076,8 @@ static inline uint8_t get_container_type( container_t *container_clone(const container_t *container, uint8_t typecode); /* access to container underneath, cloning it if needed */ -static inline container_t *get_writable_copy_if_shared( - container_t *c, uint8_t *type -){ +static inline container_t *get_writable_copy_if_shared(container_t *c, + uint8_t *type) { if (*type == SHARED_CONTAINER_TYPE) { // shared, return enclosed container return shared_container_extract_copy(CAST_shared(c), type); } else { @@ -4011,9 +4097,8 @@ static const char *shared_container_names[] = { // if a new container is produced, caller responsible for freeing the previous // one // container should not be a shared container -static inline bitset_container_t *container_to_bitset( - container_t *c, uint8_t typecode -){ +static inline bitset_container_t *container_to_bitset(container_t *c, + uint8_t typecode) { bitset_container_t *result = NULL; switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4054,9 +4139,8 @@ static inline bitset_container_t *container_to_bitset( } }*/ -static inline const char *get_full_container_name( - const container_t *c, uint8_t typecode -){ +static inline const char *get_full_container_name(const container_t *c, + uint8_t typecode) { switch (typecode) { case BITSET_CONTAINER_TYPE: return container_names[0]; @@ -4090,9 +4174,8 @@ static inline const char *get_full_container_name( /** * Get the container cardinality (number of elements), requires a typecode */ -static inline int container_get_cardinality( - const container_t *c, uint8_t typecode -){ +static inline int container_get_cardinality(const container_t *c, + uint8_t typecode) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4107,8 +4190,6 @@ static inline int container_get_cardinality( return 0; // unreached } - - // returns true if a container is known to be full. Note that a lazy bitset // container // might be full without us knowing @@ -4116,11 +4197,11 @@ static inline bool container_is_full(const container_t *c, uint8_t typecode) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: - return bitset_container_cardinality( - const_CAST_bitset(c)) == (1 << 16); + return bitset_container_cardinality(const_CAST_bitset(c)) == + (1 << 16); case ARRAY_CONTAINER_TYPE: - return array_container_cardinality( - const_CAST_array(c)) == (1 << 16); + return array_container_cardinality(const_CAST_array(c)) == + (1 << 16); case RUN_CONTAINER_TYPE: return run_container_is_full(const_CAST_run(c)); } @@ -4129,9 +4210,7 @@ static inline bool container_is_full(const container_t *c, uint8_t typecode) { return 0; // unreached } -static inline int container_shrink_to_fit( - container_t *c, uint8_t type -){ +static inline int container_shrink_to_fit(container_t *c, uint8_t type) { c = container_mutable_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -4146,41 +4225,36 @@ static inline int container_shrink_to_fit( return 0; // unreached } - /** * make a container with a run of ones */ /* initially always use a run container, even if an array might be * marginally * smaller */ -static inline container_t *container_range_of_ones( - uint32_t range_start, uint32_t range_end, - uint8_t *result_type -){ +static inline container_t *container_range_of_ones(uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { assert(range_end >= range_start); - uint64_t cardinality = range_end - range_start + 1; - if(cardinality <= 2) { - *result_type = ARRAY_CONTAINER_TYPE; - return array_container_create_range(range_start, range_end); + uint64_t cardinality = range_end - range_start + 1; + if (cardinality <= 2) { + *result_type = ARRAY_CONTAINER_TYPE; + return array_container_create_range(range_start, range_end); } else { - *result_type = RUN_CONTAINER_TYPE; - return run_container_create_range(range_start, range_end); + *result_type = RUN_CONTAINER_TYPE; + return run_container_create_range(range_start, range_end); } } - /* Create a container with all the values between in [min,max) at a distance k*step from min. */ -static inline container_t *container_from_range( - uint8_t *type, uint32_t min, - uint32_t max, uint16_t step -){ +static inline container_t *container_from_range(uint8_t *type, uint32_t min, + uint32_t max, uint16_t step) { if (step == 0) return NULL; // being paranoid if (step == 1) { - return container_range_of_ones(min,max,type); + return container_range_of_ones(min, max, type); // Note: the result is not always a run (need to check the cardinality) //*type = RUN_CONTAINER_TYPE; - //return run_container_create_range(min, max); + // return run_container_create_range(min, max); } int size = (max - min + step - 1) / step; if (size <= DEFAULT_MAX_SIZE) { // array container @@ -4201,9 +4275,8 @@ static inline container_t *container_from_range( /** * "repair" the container after lazy operations. */ -static inline container_t *container_repair_after_lazy( - container_t *c, uint8_t *type -){ +static inline container_t *container_repair_after_lazy(container_t *c, + uint8_t *type) { c = get_writable_copy_if_shared(c, type); // !!! unnecessary cloning container_t *result = NULL; switch (*type) { @@ -4216,12 +4289,13 @@ static inline container_t *container_repair_after_lazy( *type = ARRAY_CONTAINER_TYPE; return result; } - return c; } + return c; + } case ARRAY_CONTAINER_TYPE: return c; // nothing to do case RUN_CONTAINER_TYPE: - return convert_run_to_efficient_container_and_free( - CAST_run(c), type); + return convert_run_to_efficient_container_and_free(CAST_run(c), + type); case SHARED_CONTAINER_TYPE: assert(false); } @@ -4238,10 +4312,8 @@ static inline container_t *container_repair_after_lazy( * container_write(container, buf). * */ -static inline int32_t container_write( - const container_t *c, uint8_t typecode, - char *buf -){ +static inline int32_t container_write(const container_t *c, uint8_t typecode, + char *buf) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4261,9 +4333,8 @@ static inline int32_t container_write( * container_write), requires a * typecode */ -static inline int32_t container_size_in_bytes( - const container_t *c, uint8_t typecode -){ +static inline int32_t container_size_in_bytes(const container_t *c, + uint8_t typecode) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4290,20 +4361,19 @@ void container_printf(const container_t *container, uint8_t typecode); void container_printf_as_uint32_array(const container_t *container, uint8_t typecode, uint32_t base); -bool container_internal_validate(const container_t *container, - uint8_t typecode, const char **reason); +bool container_internal_validate(const container_t *container, uint8_t typecode, + const char **reason); /** * Checks whether a container is not empty, requires a typecode */ -static inline bool container_nonzero_cardinality( - const container_t *c, uint8_t typecode -){ +static inline bool container_nonzero_cardinality(const container_t *c, + uint8_t typecode) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: return bitset_container_const_nonzero_cardinality( - const_CAST_bitset(c)); + const_CAST_bitset(c)); case ARRAY_CONTAINER_TYPE: return array_container_nonzero_cardinality(const_CAST_array(c)); case RUN_CONTAINER_TYPE: @@ -4324,22 +4394,20 @@ void container_free(container_t *container, uint8_t typecode); * "base" (most significant values) * Returns number of ints added. */ -static inline int container_to_uint32_array( - uint32_t *output, - const container_t *c, uint8_t typecode, - uint32_t base -){ +static inline int container_to_uint32_array(uint32_t *output, + const container_t *c, + uint8_t typecode, uint32_t base) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: - return bitset_container_to_uint32_array( - output, const_CAST_bitset(c), base); + return bitset_container_to_uint32_array(output, + const_CAST_bitset(c), base); case ARRAY_CONTAINER_TYPE: - return array_container_to_uint32_array( - output, const_CAST_array(c), base); + return array_container_to_uint32_array(output, const_CAST_array(c), + base); case RUN_CONTAINER_TYPE: - return run_container_to_uint32_array( - output, const_CAST_run(c), base); + return run_container_to_uint32_array(output, const_CAST_run(c), + base); } assert(false); roaring_unreachable; @@ -4355,8 +4423,7 @@ static inline int container_to_uint32_array( static inline container_t *container_add( container_t *c, uint16_t val, uint8_t typecode, // !!! should be second argument? - uint8_t *new_typecode -){ + uint8_t *new_typecode) { c = get_writable_copy_if_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4369,7 +4436,7 @@ static inline container_t *container_add( *new_typecode = ARRAY_CONTAINER_TYPE; return ac; } else { - bitset_container_t* bitset = bitset_container_from_array(ac); + bitset_container_t *bitset = bitset_container_from_array(ac); bitset_container_add(bitset, val); *new_typecode = BITSET_CONTAINER_TYPE; return bitset; @@ -4397,8 +4464,7 @@ static inline container_t *container_add( static inline container_t *container_remove( container_t *c, uint16_t val, uint8_t typecode, // !!! should be second argument? - uint8_t *new_typecode -){ + uint8_t *new_typecode) { c = get_writable_copy_if_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4431,10 +4497,9 @@ static inline container_t *container_remove( * Check whether a value is in a container, requires a typecode */ static inline bool container_contains( - const container_t *c, - uint16_t val, + const container_t *c, uint16_t val, uint8_t typecode // !!! should be second argument? -){ +) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: @@ -4451,25 +4516,24 @@ static inline bool container_contains( } /** - * Check whether a range of values from range_start (included) to range_end (excluded) - * is in a container, requires a typecode + * Check whether a range of values from range_start (included) to range_end + * (excluded) is in a container, requires a typecode */ static inline bool container_contains_range( - const container_t *c, - uint32_t range_start, uint32_t range_end, + const container_t *c, uint32_t range_start, uint32_t range_end, uint8_t typecode // !!! should be second argument? -){ +) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: - return bitset_container_get_range(const_CAST_bitset(c), - range_start, range_end); + return bitset_container_get_range(const_CAST_bitset(c), range_start, + range_end); case ARRAY_CONTAINER_TYPE: return array_container_contains_range(const_CAST_array(c), - range_start, range_end); + range_start, range_end); case RUN_CONTAINER_TYPE: - return run_container_contains_range(const_CAST_run(c), - range_start, range_end); + return run_container_contains_range(const_CAST_run(c), range_start, + range_end); default: assert(false); roaring_unreachable; @@ -4481,50 +4545,47 @@ static inline bool container_contains_range( * Returns true if the two containers have the same content. Note that * two containers having different types can be "equal" in this sense. */ -static inline bool container_equals( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2 -){ +static inline bool container_equals(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): return bitset_container_equals(const_CAST_bitset(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): return run_container_equals_bitset(const_CAST_run(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): return run_container_equals_bitset(const_CAST_run(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): // java would always return false? return array_container_equal_bitset(const_CAST_array(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): // java would always return false? return array_container_equal_bitset(const_CAST_array(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): return run_container_equals_array(const_CAST_run(c2), const_CAST_array(c1)); - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): return run_container_equals_array(const_CAST_run(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): return array_container_equals(const_CAST_array(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(RUN,RUN): - return run_container_equals(const_CAST_run(c1), - const_CAST_run(c2)); + case CONTAINER_PAIR(RUN, RUN): + return run_container_equals(const_CAST_run(c1), const_CAST_run(c2)); default: assert(false); @@ -4537,45 +4598,43 @@ static inline bool container_equals( * Returns true if the container c1 is a subset of the container c2. Note that * c1 can be a subset of c2 even if they have a different type. */ -static inline bool container_is_subset( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2 -){ +static inline bool container_is_subset(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): return bitset_container_is_subset(const_CAST_bitset(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): return bitset_container_is_subset_run(const_CAST_bitset(c1), const_CAST_run(c2)); - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): return run_container_is_subset_bitset(const_CAST_run(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): return false; // by construction, size(c1) > size(c2) - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): return array_container_is_subset_bitset(const_CAST_array(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): return array_container_is_subset_run(const_CAST_array(c1), const_CAST_run(c2)); - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): return run_container_is_subset_array(const_CAST_run(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): return array_container_is_subset(const_CAST_array(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): return run_container_is_subset(const_CAST_run(c1), const_CAST_run(c2)); @@ -4593,40 +4652,36 @@ static inline bool container_is_subset( * type result_type), requires a typecode. This allocates new memory, caller * is responsible for deallocation. */ -static inline container_t *container_and( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_and(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - *result_type = bitset_bitset_container_intersection( - const_CAST_bitset(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_intersection( + const_CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): result = array_container_create(); - array_container_intersection(const_CAST_array(c1), - const_CAST_array(c2), - CAST_array(result)); + array_container_intersection( + const_CAST_array(c1), const_CAST_array(c2), CAST_array(result)); *result_type = ARRAY_CONTAINER_TYPE; // never bitset return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): result = run_container_create(); - run_container_intersection(const_CAST_run(c1), - const_CAST_run(c2), + run_container_intersection(const_CAST_run(c1), const_CAST_run(c2), CAST_run(result)); - return convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + return convert_run_to_efficient_container_and_free(CAST_run(result), + result_type); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): result = array_container_create(); array_bitset_container_intersection(const_CAST_array(c2), const_CAST_bitset(c1), @@ -4634,7 +4689,7 @@ static inline container_t *container_and( *result_type = ARRAY_CONTAINER_TYPE; // never bitset return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; // never bitset array_bitset_container_intersection(const_CAST_array(c1), @@ -4642,36 +4697,34 @@ static inline container_t *container_and( CAST_array(result)); return result; - case CONTAINER_PAIR(BITSET,RUN): - *result_type = run_bitset_container_intersection( - const_CAST_run(c2), - const_CAST_bitset(c1), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, RUN): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c2), const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): - *result_type = run_bitset_container_intersection( - const_CAST_run(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; // never bitset - array_run_container_intersection(const_CAST_array(c1), - const_CAST_run(c2), - CAST_array(result)); + array_run_container_intersection( + const_CAST_array(c1), const_CAST_run(c2), CAST_array(result)); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; // never bitset - array_run_container_intersection(const_CAST_array(c2), - const_CAST_run(c1), - CAST_array(result)); + array_run_container_intersection( + const_CAST_array(c2), const_CAST_run(c1), CAST_array(result)); return result; default: @@ -4684,46 +4737,46 @@ static inline container_t *container_and( /** * Compute the size of the intersection between two containers. */ -static inline int container_and_cardinality( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2 -){ +static inline int container_and_cardinality(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - return bitset_container_and_justcard( - const_CAST_bitset(c1), const_CAST_bitset(c2)); + case CONTAINER_PAIR(BITSET, BITSET): + return bitset_container_and_justcard(const_CAST_bitset(c1), + const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): return array_container_intersection_cardinality( const_CAST_array(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(RUN,RUN): - return run_container_intersection_cardinality( - const_CAST_run(c1), const_CAST_run(c2)); + case CONTAINER_PAIR(RUN, RUN): + return run_container_intersection_cardinality(const_CAST_run(c1), + const_CAST_run(c2)); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): return array_bitset_container_intersection_cardinality( const_CAST_array(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): return array_bitset_container_intersection_cardinality( const_CAST_array(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): return run_bitset_container_intersection_cardinality( const_CAST_run(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): return run_bitset_container_intersection_cardinality( const_CAST_run(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): return array_run_container_intersection_cardinality( const_CAST_array(c1), const_CAST_run(c2)); - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): return array_run_container_intersection_cardinality( const_CAST_array(c2), const_CAST_run(c1)); @@ -4737,46 +4790,44 @@ static inline int container_and_cardinality( /** * Check whether two containers intersect. */ -static inline bool container_intersect( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2 -){ +static inline bool container_intersect(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): return bitset_container_intersect(const_CAST_bitset(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): return array_container_intersect(const_CAST_array(c1), const_CAST_array(c2)); - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): return run_container_intersect(const_CAST_run(c1), const_CAST_run(c2)); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): return array_bitset_container_intersect(const_CAST_array(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): return array_bitset_container_intersect(const_CAST_array(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): return run_bitset_container_intersect(const_CAST_run(c2), const_CAST_bitset(c1)); - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): return run_bitset_container_intersect(const_CAST_run(c1), const_CAST_bitset(c2)); - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): return array_run_container_intersect(const_CAST_array(c1), const_CAST_run(c2)); - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): return array_run_container_intersect(const_CAST_array(c2), const_CAST_run(c1)); @@ -4796,40 +4847,36 @@ static inline bool container_intersect( The type of the first container may change. Returns the modified (and possibly new) container. */ -static inline container_t *container_iand( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_iand(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = get_writable_copy_if_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - *result_type = - bitset_bitset_container_intersection_inplace( - CAST_bitset(c1), const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = bitset_bitset_container_intersection_inplace( + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): array_container_intersection_inplace(CAST_array(c1), const_CAST_array(c2)); *result_type = ARRAY_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): result = run_container_create(); - run_container_intersection(const_CAST_run(c1), - const_CAST_run(c2), + run_container_intersection(const_CAST_run(c1), const_CAST_run(c2), CAST_run(result)); // as of January 2016, Java code used non-in-place intersection for // two runcontainers - return convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + return convert_run_to_efficient_container_and_free(CAST_run(result), + result_type); - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): // c1 is a bitmap so no inplace possible result = array_container_create(); array_bitset_container_intersection(const_CAST_array(c2), @@ -4838,44 +4885,41 @@ static inline container_t *container_iand( *result_type = ARRAY_CONTAINER_TYPE; // never bitset return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): *result_type = ARRAY_CONTAINER_TYPE; // never bitset array_bitset_container_intersection( - const_CAST_array(c1), const_CAST_bitset(c2), - CAST_array(c1)); // result is allowed to be same as c1 + const_CAST_array(c1), const_CAST_bitset(c2), + CAST_array(c1)); // result is allowed to be same as c1 return c1; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): // will attempt in-place computation *result_type = run_bitset_container_intersection( - const_CAST_run(c2), - const_CAST_bitset(c1), &c1) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + const_CAST_run(c2), const_CAST_bitset(c1), &c1) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(RUN,BITSET): - *result_type = run_bitset_container_intersection( - const_CAST_run(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; // never bitset - array_run_container_intersection(const_CAST_array(c1), - const_CAST_run(c2), - CAST_array(result)); + array_run_container_intersection( + const_CAST_array(c1), const_CAST_run(c2), CAST_array(result)); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; // never bitset - array_run_container_intersection(const_CAST_array(c2), - const_CAST_run(c1), - CAST_array(result)); + array_run_container_intersection( + const_CAST_array(c2), const_CAST_run(c1), CAST_array(result)); return result; default: @@ -4890,43 +4934,39 @@ static inline container_t *container_iand( * result_type), requires a typecode. This allocates new memory, caller * is responsible for deallocation. */ -static inline container_t *container_or( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_or(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): result = bitset_container_create(); - bitset_container_or(const_CAST_bitset(c1), - const_CAST_bitset(c2), + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): - *result_type = array_array_container_union( - const_CAST_array(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_union(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): result = run_container_create(); - run_container_union(const_CAST_run(c1), - const_CAST_run(c2), + run_container_union(const_CAST_run(c1), const_CAST_run(c2), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // todo: could be optimized since will never convert to array result = convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + CAST_run(result), result_type); return result; - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): result = bitset_container_create(); array_bitset_container_union(const_CAST_array(c2), const_CAST_bitset(c1), @@ -4934,7 +4974,7 @@ static inline container_t *container_or( *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): result = bitset_container_create(); array_bitset_container_union(const_CAST_array(c1), const_CAST_bitset(c2), @@ -4942,52 +4982,46 @@ static inline container_t *container_or( *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; - run_container_copy(const_CAST_run(c2), - CAST_run(result)); + run_container_copy(const_CAST_run(c2), CAST_run(result)); return result; } result = bitset_container_create(); - run_bitset_container_union(const_CAST_run(c2), - const_CAST_bitset(c1), - CAST_bitset(result)); + run_bitset_container_union( + const_CAST_run(c2), const_CAST_bitset(c1), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): if (run_container_is_full(const_CAST_run(c1))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; - run_container_copy(const_CAST_run(c1), - CAST_run(result)); + run_container_copy(const_CAST_run(c1), CAST_run(result)); return result; } result = bitset_container_create(); - run_bitset_container_union(const_CAST_run(c1), - const_CAST_bitset(c2), - CAST_bitset(result)); + run_bitset_container_union( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = run_container_create(); - array_run_container_union(const_CAST_array(c1), - const_CAST_run(c2), + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), CAST_run(result)); result = convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + CAST_run(result), result_type); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): result = run_container_create(); - array_run_container_union(const_CAST_array(c2), - const_CAST_run(c1), + array_run_container_union(const_CAST_array(c2), const_CAST_run(c1), CAST_run(result)); result = convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + CAST_run(result), result_type); return result; default: @@ -5005,35 +5039,34 @@ static inline container_t *container_or( * This lazy version delays some operations such as the maintenance of the * cardinality. It requires repair later on the generated containers. */ -static inline container_t *container_lazy_or( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_lazy_or(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): result = bitset_container_create(); - bitset_container_or_nocard( - const_CAST_bitset(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // is lazy + bitset_container_or_nocard(const_CAST_bitset(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): - *result_type = array_array_container_lazy_union( - const_CAST_array(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_lazy_union(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): result = run_container_create(); - run_container_union(const_CAST_run(c1), - const_CAST_run(c2), + run_container_union(const_CAST_run(c1), const_CAST_run(c2), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // we are being lazy @@ -5041,23 +5074,23 @@ static inline container_t *container_lazy_or( CAST_run(result), result_type); return result; - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): result = bitset_container_create(); - array_bitset_container_lazy_union( - const_CAST_array(c2), const_CAST_bitset(c1), - CAST_bitset(result)); // is lazy + array_bitset_container_lazy_union(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): result = bitset_container_create(); - array_bitset_container_lazy_union( - const_CAST_array(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // is lazy + array_bitset_container_lazy_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; @@ -5065,13 +5098,13 @@ static inline container_t *container_lazy_or( return result; } result = bitset_container_create(); - run_bitset_container_lazy_union( - const_CAST_run(c2), const_CAST_bitset(c1), - CAST_bitset(result)); // is lazy + run_bitset_container_lazy_union(const_CAST_run(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): if (run_container_is_full(const_CAST_run(c1))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; @@ -5079,27 +5112,25 @@ static inline container_t *container_lazy_or( return result; } result = bitset_container_create(); - run_bitset_container_lazy_union( - const_CAST_run(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // is lazy + run_bitset_container_lazy_union(const_CAST_run(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = run_container_create(); - array_run_container_union(const_CAST_array(c1), - const_CAST_run(c2), + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // next line skipped since we are lazy // result = convert_run_to_efficient_container(result, result_type); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): result = run_container_create(); - array_run_container_union( - const_CAST_array(c2), const_CAST_run(c1), - CAST_run(result)); // TODO make lazy + array_run_container_union(const_CAST_array(c2), const_CAST_run(c1), + CAST_run(result)); // TODO make lazy *result_type = RUN_CONTAINER_TYPE; // next line skipped since we are lazy // result = convert_run_to_efficient_container(result, result_type); @@ -5120,19 +5151,16 @@ static inline container_t *container_lazy_or( * created and the caller is responsible for freeing it. * The type of the first container may change. Returns the modified * (and possibly new) container -*/ -static inline container_t *container_ior( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ + */ +static inline container_t *container_ior(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = get_writable_copy_if_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - bitset_container_or(const_CAST_bitset(c1), - const_CAST_bitset(c2), + case CONTAINER_PAIR(BITSET, BITSET): + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(c1)); #ifdef OR_BITSET_CONVERSION_TO_FULL if (CAST_bitset(c1)->cardinality == (1 << 16)) { // we convert @@ -5144,30 +5172,28 @@ static inline container_t *container_ior( *result_type = BITSET_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): *result_type = array_array_container_inplace_union( - CAST_array(c1), const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; - if((result == NULL) - && (*result_type == ARRAY_CONTAINER_TYPE)) { - return c1; // the computation was done in-place! + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + if ((result == NULL) && (*result_type == ARRAY_CONTAINER_TYPE)) { + return c1; // the computation was done in-place! } return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): run_container_union_inplace(CAST_run(c1), const_CAST_run(c2)); return convert_run_to_efficient_container(CAST_run(c1), result_type); - case CONTAINER_PAIR(BITSET,ARRAY): - array_bitset_container_union(const_CAST_array(c2), - const_CAST_bitset(c1), - CAST_bitset(c1)); + case CONTAINER_PAIR(BITSET, ARRAY): + array_bitset_container_union( + const_CAST_array(c2), const_CAST_bitset(c1), CAST_bitset(c1)); *result_type = BITSET_CONTAINER_TYPE; // never array return c1; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): // c1 is an array, so no in-place possible result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; @@ -5176,7 +5202,7 @@ static inline container_t *container_ior( CAST_bitset(result)); return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; @@ -5189,32 +5215,29 @@ static inline container_t *container_ior( *result_type = BITSET_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): if (run_container_is_full(const_CAST_run(c1))) { *result_type = RUN_CONTAINER_TYPE; return c1; } result = bitset_container_create(); - run_bitset_container_union(const_CAST_run(c1), - const_CAST_bitset(c2), - CAST_bitset(result)); + run_bitset_container_union( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = run_container_create(); - array_run_container_union(const_CAST_array(c1), - const_CAST_run(c2), + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), CAST_run(result)); result = convert_run_to_efficient_container_and_free( - CAST_run(result), result_type); + CAST_run(result), result_type); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): array_run_container_inplace_union(const_CAST_array(c2), CAST_run(c1)); - c1 = convert_run_to_efficient_container(CAST_run(c1), - result_type); + c1 = convert_run_to_efficient_container(CAST_run(c1), result_type); return c1; default: @@ -5235,22 +5258,20 @@ static inline container_t *container_ior( * * This lazy version delays some operations such as the maintenance of the * cardinality. It requires repair later on the generated containers. -*/ -static inline container_t *container_lazy_ior( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ + */ +static inline container_t *container_lazy_ior(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { assert(type1 != SHARED_CONTAINER_TYPE); // c1 = get_writable_copy_if_shared(c1,&type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): #ifdef LAZY_OR_BITSET_CONVERSION_TO_FULL // if we have two bitsets, we might as well compute the cardinality - bitset_container_or(const_CAST_bitset(c1), - const_CAST_bitset(c2), + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(c1)); // it is possible that two bitsets can lead to a full container if (CAST_bitset(c1)->cardinality == (1 << 16)) { // we convert @@ -5260,54 +5281,49 @@ static inline container_t *container_lazy_ior( } #else bitset_container_or_nocard(const_CAST_bitset(c1), - const_CAST_bitset(c2), - CAST_bitset(c1)); + const_CAST_bitset(c2), CAST_bitset(c1)); #endif *result_type = BITSET_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): *result_type = array_array_container_lazy_inplace_union( - CAST_array(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; - if((result == NULL) - && (*result_type == ARRAY_CONTAINER_TYPE)) { - return c1; // the computation was done in-place! + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + if ((result == NULL) && (*result_type == ARRAY_CONTAINER_TYPE)) { + return c1; // the computation was done in-place! } return result; - case CONTAINER_PAIR(RUN,RUN): - run_container_union_inplace(CAST_run(c1), - const_CAST_run(c2)); + case CONTAINER_PAIR(RUN, RUN): + run_container_union_inplace(CAST_run(c1), const_CAST_run(c2)); *result_type = RUN_CONTAINER_TYPE; return convert_run_to_efficient_container(CAST_run(c1), result_type); - case CONTAINER_PAIR(BITSET,ARRAY): - array_bitset_container_lazy_union( - const_CAST_array(c2), const_CAST_bitset(c1), - CAST_bitset(c1)); // is lazy - *result_type = BITSET_CONTAINER_TYPE; // never array + case CONTAINER_PAIR(BITSET, ARRAY): + array_bitset_container_lazy_union(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(c1)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; // never array return c1; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): // c1 is an array, so no in-place possible result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; - array_bitset_container_lazy_union( - const_CAST_array(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // is lazy + array_bitset_container_lazy_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = run_container_create(); *result_type = RUN_CONTAINER_TYPE; - run_container_copy(const_CAST_run(c2), - CAST_run(result)); + run_container_copy(const_CAST_run(c2), CAST_run(result)); return result; } run_bitset_container_lazy_union( @@ -5316,22 +5332,21 @@ static inline container_t *container_lazy_ior( *result_type = BITSET_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): if (run_container_is_full(const_CAST_run(c1))) { *result_type = RUN_CONTAINER_TYPE; return c1; } result = bitset_container_create(); - run_bitset_container_lazy_union( - const_CAST_run(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // lazy + run_bitset_container_lazy_union(const_CAST_run(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = run_container_create(); - array_run_container_union(const_CAST_array(c1), - const_CAST_run(c2), + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // next line skipped since we are lazy @@ -5339,7 +5354,7 @@ static inline container_t *container_lazy_ior( // result_type); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): array_run_container_inplace_union(const_CAST_array(c2), CAST_run(c1)); *result_type = RUN_CONTAINER_TYPE; @@ -5360,79 +5375,74 @@ static inline container_t *container_lazy_ior( * container (having type result_type), requires a typecode. This allocates new * memory, caller is responsible for deallocation. */ -static inline container_t* container_xor( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_xor(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - *result_type = bitset_bitset_container_xor( - const_CAST_bitset(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; - return result; - - case CONTAINER_PAIR(ARRAY,ARRAY): - *result_type = array_array_container_xor( - const_CAST_array(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_xor(const_CAST_bitset(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(ARRAY, ARRAY): *result_type = - run_run_container_xor(const_CAST_run(c1), - const_CAST_run(c2), &result); + array_array_container_xor(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,ARRAY): - *result_type = array_bitset_container_xor( - const_CAST_array(c2), - const_CAST_bitset(c1), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_xor( + const_CAST_run(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(ARRAY,BITSET): - *result_type = array_bitset_container_xor( - const_CAST_array(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = + array_bitset_container_xor(const_CAST_array(c2), + const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,RUN): - *result_type = run_bitset_container_xor( - const_CAST_run(c2), - const_CAST_bitset(c1), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(ARRAY, BITSET): + *result_type = + array_bitset_container_xor(const_CAST_array(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): - *result_type = run_bitset_container_xor( - const_CAST_run(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, RUN): + *result_type = + run_bitset_container_xor(const_CAST_run(c2), + const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(RUN, BITSET): *result_type = - array_run_container_xor(const_CAST_array(c1), - const_CAST_run(c2), &result); + run_bitset_container_xor(const_CAST_run(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,ARRAY): - *result_type = - array_run_container_xor(const_CAST_array(c2), - const_CAST_run(c1), &result); + case CONTAINER_PAIR(ARRAY, RUN): + *result_type = (uint8_t)array_run_container_xor( + const_CAST_array(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)array_run_container_xor( + const_CAST_array(c2), const_CAST_run(c1), &result); return result; default: @@ -5444,14 +5454,13 @@ static inline container_t* container_xor( /* Applies an offset to the non-empty container 'c'. * The results are stored in new containers returned via 'lo' and 'hi', for the - * low and high halves of the result (where the low half matches the original key - * and the high one corresponds to values for the following key). - * Either one of 'lo' and 'hi' are allowed to be 'NULL', but not both. - * Whenever one of them is not 'NULL', it should point to a 'NULL' container. - * Whenever one of them is 'NULL' the shifted elements for that part will not be - * computed. - * If either of the resulting containers turns out to be empty, the pointed - * container will remain 'NULL'. + * low and high halves of the result (where the low half matches the original + * key and the high one corresponds to values for the following key). Either one + * of 'lo' and 'hi' are allowed to be 'NULL', but not both. Whenever one of them + * is not 'NULL', it should point to a 'NULL' container. Whenever one of them is + * 'NULL' the shifted elements for that part will not be computed. If either of + * the resulting containers turns out to be empty, the pointed container will + * remain 'NULL'. */ static inline void container_add_offset(const container_t *c, uint8_t type, container_t **lo, container_t **hi, @@ -5463,19 +5472,19 @@ static inline void container_add_offset(const container_t *c, uint8_t type, assert(hi == NULL || *hi == NULL); switch (type) { - case BITSET_CONTAINER_TYPE: - bitset_container_offset(const_CAST_bitset(c), lo, hi, offset); - break; - case ARRAY_CONTAINER_TYPE: - array_container_offset(const_CAST_array(c), lo, hi, offset); - break; - case RUN_CONTAINER_TYPE: - run_container_offset(const_CAST_run(c), lo, hi, offset); - break; - default: - assert(false); - roaring_unreachable; - break; + case BITSET_CONTAINER_TYPE: + bitset_container_offset(const_CAST_bitset(c), lo, hi, offset); + break; + case ARRAY_CONTAINER_TYPE: + array_container_offset(const_CAST_array(c), lo, hi, offset); + break; + case RUN_CONTAINER_TYPE: + run_container_offset(const_CAST_run(c), lo, hi, offset); + break; + default: + assert(false); + roaring_unreachable; + break; } } @@ -5487,39 +5496,38 @@ static inline void container_add_offset(const container_t *c, uint8_t type, * This lazy version delays some operations such as the maintenance of the * cardinality. It requires repair later on the generated containers. */ -static inline container_t *container_lazy_xor( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_lazy_xor(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): result = bitset_container_create(); - bitset_container_xor_nocard( - const_CAST_bitset(c1), const_CAST_bitset(c2), - CAST_bitset(result)); // is lazy + bitset_container_xor_nocard(const_CAST_bitset(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): - *result_type = array_array_container_lazy_xor( - const_CAST_array(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_lazy_xor(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): // nothing special done yet. - *result_type = - run_run_container_xor(const_CAST_run(c1), - const_CAST_run(c2), &result); + *result_type = (uint8_t)run_run_container_xor( + const_CAST_run(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; array_bitset_container_lazy_xor(const_CAST_array(c2), @@ -5527,7 +5535,7 @@ static inline container_t *container_lazy_xor( CAST_bitset(result)); return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; array_bitset_container_lazy_xor(const_CAST_array(c1), @@ -5535,37 +5543,33 @@ static inline container_t *container_lazy_xor( CAST_bitset(result)); return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): result = bitset_container_create(); - run_bitset_container_lazy_xor(const_CAST_run(c2), - const_CAST_bitset(c1), - CAST_bitset(result)); + run_bitset_container_lazy_xor( + const_CAST_run(c2), const_CAST_bitset(c1), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): result = bitset_container_create(); - run_bitset_container_lazy_xor(const_CAST_run(c1), - const_CAST_bitset(c2), - CAST_bitset(result)); + run_bitset_container_lazy_xor( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); *result_type = BITSET_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): result = run_container_create(); array_run_container_lazy_xor(const_CAST_array(c1), - const_CAST_run(c2), - CAST_run(result)); + const_CAST_run(c2), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // next line skipped since we are lazy // result = convert_run_to_efficient_container(result, result_type); return result; - case CONTAINER_PAIR(RUN,ARRAY): + case CONTAINER_PAIR(RUN, ARRAY): result = run_container_create(); array_run_container_lazy_xor(const_CAST_array(c2), - const_CAST_run(c1), - CAST_run(result)); + const_CAST_run(c1), CAST_run(result)); *result_type = RUN_CONTAINER_TYPE; // next line skipped since we are lazy // result = convert_run_to_efficient_container(result, result_type); @@ -5583,76 +5587,73 @@ static inline container_t *container_lazy_xor( * If the returned pointer is identical to c1, then the container has been * modified. * If the returned pointer is different from c1, then a new container has been - * created and the caller is responsible for freeing it. - * The type of the first container may change. Returns the modified - * (and possibly new) container -*/ -static inline container_t *container_ixor( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ + * created. The original container is freed by container_ixor. + * The type of the first container may change. Returns the modified (and + * possibly new) container. + */ +static inline container_t *container_ixor(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { c1 = get_writable_copy_if_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): *result_type = bitset_bitset_container_ixor( - CAST_bitset(c1), const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): *result_type = array_array_container_ixor( - CAST_array(c1), const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): - *result_type = run_run_container_ixor( + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_ixor( CAST_run(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): *result_type = bitset_array_container_ixor( - CAST_bitset(c1), const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_bitset(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): *result_type = array_bitset_container_ixor( - CAST_array(c1), const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_array(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,RUN): - *result_type = - bitset_run_container_ixor( - CAST_bitset(c1), const_CAST_run(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, RUN): + *result_type = bitset_run_container_ixor( + CAST_bitset(c1), const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): *result_type = run_bitset_container_ixor( - CAST_run(c1), const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): - *result_type = array_run_container_ixor( - CAST_array(c1), const_CAST_run(c2), &result); + case CONTAINER_PAIR(ARRAY, RUN): + *result_type = (uint8_t)array_run_container_ixor( + CAST_array(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(RUN,ARRAY): - *result_type = run_array_container_ixor( - CAST_run(c1), const_CAST_array(c2), &result); + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_ixor( + CAST_run(c1), const_CAST_array(c2), &result); return result; default: @@ -5673,19 +5674,17 @@ static inline container_t *container_ixor( * * This lazy version delays some operations such as the maintenance of the * cardinality. It requires repair later on the generated containers. -*/ -static inline container_t *container_lazy_ixor( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ + */ +static inline container_t *container_lazy_ixor(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { assert(type1 != SHARED_CONTAINER_TYPE); // c1 = get_writable_copy_if_shared(c1,&type1); c2 = container_unwrap_shared(c2, &type2); switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - bitset_container_xor_nocard(CAST_bitset(c1), - const_CAST_bitset(c2), + case CONTAINER_PAIR(BITSET, BITSET): + bitset_container_xor_nocard(CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(c1)); // is lazy *result_type = BITSET_CONTAINER_TYPE; return c1; @@ -5710,51 +5709,49 @@ static inline container_t *container_lazy_ixor( * container (having type result_type), requires a typecode. This allocates new * memory, caller is responsible for deallocation. */ -static inline container_t *container_andnot( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ +static inline container_t *container_andnot(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { c1 = container_unwrap_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): - *result_type = bitset_bitset_container_andnot( - const_CAST_bitset(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_andnot(const_CAST_bitset(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): + case CONTAINER_PAIR(ARRAY, ARRAY): result = array_container_create(); - array_array_container_andnot(const_CAST_array(c1), - const_CAST_array(c2), - CAST_array(result)); + array_array_container_andnot( + const_CAST_array(c1), const_CAST_array(c2), CAST_array(result)); *result_type = ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,RUN): + case CONTAINER_PAIR(RUN, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; return result; } - *result_type = - run_run_container_andnot(const_CAST_run(c1), - const_CAST_run(c2), &result); + *result_type = (uint8_t)run_run_container_andnot( + const_CAST_run(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(BITSET,ARRAY): - *result_type = bitset_array_container_andnot( - const_CAST_bitset(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = + bitset_array_container_andnot(const_CAST_bitset(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): result = array_container_create(); array_bitset_container_andnot(const_CAST_array(c1), const_CAST_bitset(c2), @@ -5762,44 +5759,42 @@ static inline container_t *container_andnot( *result_type = ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; return result; } - *result_type = bitset_run_container_andnot( - const_CAST_bitset(c1), - const_CAST_run(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + *result_type = + bitset_run_container_andnot(const_CAST_bitset(c1), + const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): - *result_type = run_bitset_container_andnot( - const_CAST_run(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_andnot(const_CAST_run(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): if (run_container_is_full(const_CAST_run(c2))) { result = array_container_create(); *result_type = ARRAY_CONTAINER_TYPE; return result; } result = array_container_create(); - array_run_container_andnot(const_CAST_array(c1), - const_CAST_run(c2), + array_run_container_andnot(const_CAST_array(c1), const_CAST_run(c2), CAST_array(result)); *result_type = ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,ARRAY): - *result_type = run_array_container_andnot( - const_CAST_run(c1), const_CAST_array(c2), - &result); + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_andnot( + const_CAST_run(c1), const_CAST_array(c2), &result); return result; default: @@ -5815,76 +5810,69 @@ static inline container_t *container_andnot( * If the returned pointer is identical to c1, then the container has been * modified. * If the returned pointer is different from c1, then a new container has been - * created and the caller is responsible for freeing it. - * The type of the first container may change. Returns the modified - * (and possibly new) container -*/ -static inline container_t *container_iandnot( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type -){ + * created. The original container is freed by container_iandnot. + * The type of the first container may change. Returns the modified (and + * possibly new) container. + */ +static inline container_t *container_iandnot(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { c1 = get_writable_copy_if_shared(c1, &type1); c2 = container_unwrap_shared(c2, &type2); container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { - case CONTAINER_PAIR(BITSET,BITSET): + case CONTAINER_PAIR(BITSET, BITSET): *result_type = bitset_bitset_container_iandnot( - CAST_bitset(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,ARRAY): - array_array_container_iandnot(CAST_array(c1), - const_CAST_array(c2)); + case CONTAINER_PAIR(ARRAY, ARRAY): + array_array_container_iandnot(CAST_array(c1), const_CAST_array(c2)); *result_type = ARRAY_CONTAINER_TYPE; return c1; - case CONTAINER_PAIR(RUN,RUN): - *result_type = run_run_container_iandnot( + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_iandnot( CAST_run(c1), const_CAST_run(c2), &result); return result; - case CONTAINER_PAIR(BITSET,ARRAY): + case CONTAINER_PAIR(BITSET, ARRAY): *result_type = bitset_array_container_iandnot( - CAST_bitset(c1), - const_CAST_array(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_bitset(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,BITSET): + case CONTAINER_PAIR(ARRAY, BITSET): *result_type = ARRAY_CONTAINER_TYPE; array_bitset_container_iandnot(CAST_array(c1), const_CAST_bitset(c2)); return c1; - case CONTAINER_PAIR(BITSET,RUN): + case CONTAINER_PAIR(BITSET, RUN): *result_type = bitset_run_container_iandnot( - CAST_bitset(c1), - const_CAST_run(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_bitset(c1), const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(RUN,BITSET): + case CONTAINER_PAIR(RUN, BITSET): *result_type = run_bitset_container_iandnot( - CAST_run(c1), - const_CAST_bitset(c2), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; - case CONTAINER_PAIR(ARRAY,RUN): + case CONTAINER_PAIR(ARRAY, RUN): *result_type = ARRAY_CONTAINER_TYPE; - array_run_container_iandnot(CAST_array(c1), - const_CAST_run(c2)); + array_run_container_iandnot(CAST_array(c1), const_CAST_run(c2)); return c1; - case CONTAINER_PAIR(RUN,ARRAY): - *result_type = run_array_container_iandnot( + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_iandnot( CAST_run(c1), const_CAST_array(c2), &result); return result; @@ -5900,22 +5888,20 @@ static inline container_t *container_iandnot( * to iterator. You need to specify a container and its type. * Returns true if the iteration should continue. */ -static inline bool container_iterate( - const container_t *c, uint8_t type, - uint32_t base, - roaring_iterator iterator, void *ptr -){ +static inline bool container_iterate(const container_t *c, uint8_t type, + uint32_t base, roaring_iterator iterator, + void *ptr) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: - return bitset_container_iterate(const_CAST_bitset(c), - base, iterator, ptr); + return bitset_container_iterate(const_CAST_bitset(c), base, + iterator, ptr); case ARRAY_CONTAINER_TYPE: - return array_container_iterate(const_CAST_array(c), - base, iterator, ptr); + return array_container_iterate(const_CAST_array(c), base, iterator, + ptr); case RUN_CONTAINER_TYPE: - return run_container_iterate(const_CAST_run(c), - base, iterator, ptr); + return run_container_iterate(const_CAST_run(c), base, iterator, + ptr); default: assert(false); roaring_unreachable; @@ -5925,12 +5911,10 @@ static inline bool container_iterate( return false; } -static inline bool container_iterate64( - const container_t *c, uint8_t type, - uint32_t base, - roaring_iterator64 iterator, - uint64_t high_bits, void *ptr -){ +static inline bool container_iterate64(const container_t *c, uint8_t type, + uint32_t base, + roaring_iterator64 iterator, + uint64_t high_bits, void *ptr) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -5940,8 +5924,8 @@ static inline bool container_iterate64( return array_container_iterate64(const_CAST_array(c), base, iterator, high_bits, ptr); case RUN_CONTAINER_TYPE: - return run_container_iterate64(const_CAST_run(c), base, - iterator, high_bits, ptr); + return run_container_iterate64(const_CAST_run(c), base, iterator, + high_bits, ptr); default: assert(false); roaring_unreachable; @@ -5951,28 +5935,25 @@ static inline bool container_iterate64( return false; } -static inline container_t *container_not( - const container_t *c, uint8_t type, - uint8_t *result_type -){ +static inline container_t *container_not(const container_t *c, uint8_t type, + uint8_t *result_type) { c = container_unwrap_shared(c, &type); container_t *result = NULL; switch (type) { case BITSET_CONTAINER_TYPE: - *result_type = bitset_container_negation( - const_CAST_bitset(c), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + *result_type = + bitset_container_negation(const_CAST_bitset(c), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case ARRAY_CONTAINER_TYPE: result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; - array_container_negation(const_CAST_array(c), - CAST_bitset(result)); + array_container_negation(const_CAST_array(c), CAST_bitset(result)); return result; case RUN_CONTAINER_TYPE: *result_type = - run_container_negation(const_CAST_run(c), &result); + (uint8_t)run_container_negation(const_CAST_run(c), &result); return result; default: @@ -5984,31 +5965,31 @@ static inline container_t *container_not( return NULL; } -static inline container_t *container_not_range( - const container_t *c, uint8_t type, - uint32_t range_start, uint32_t range_end, - uint8_t *result_type -){ +static inline container_t *container_not_range(const container_t *c, + uint8_t type, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { c = container_unwrap_shared(c, &type); container_t *result = NULL; switch (type) { case BITSET_CONTAINER_TYPE: *result_type = - bitset_container_negation_range( - const_CAST_bitset(c), range_start, range_end, &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + bitset_container_negation_range(const_CAST_bitset(c), + range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case ARRAY_CONTAINER_TYPE: *result_type = - array_container_negation_range( - const_CAST_array(c), range_start, range_end, &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + array_container_negation_range(const_CAST_array(c), range_start, + range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case RUN_CONTAINER_TYPE: - *result_type = run_container_negation_range( - const_CAST_run(c), range_start, range_end, &result); + *result_type = (uint8_t)run_container_negation_range( + const_CAST_run(c), range_start, range_end, &result); return result; default: @@ -6020,30 +6001,27 @@ static inline container_t *container_not_range( return NULL; } -static inline container_t *container_inot( - container_t *c, uint8_t type, - uint8_t *result_type -){ +static inline container_t *container_inot(container_t *c, uint8_t type, + uint8_t *result_type) { c = get_writable_copy_if_shared(c, &type); container_t *result = NULL; switch (type) { case BITSET_CONTAINER_TYPE: - *result_type = bitset_container_negation_inplace( - CAST_bitset(c), &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + *result_type = + bitset_container_negation_inplace(CAST_bitset(c), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case ARRAY_CONTAINER_TYPE: // will never be inplace result = bitset_container_create(); *result_type = BITSET_CONTAINER_TYPE; - array_container_negation(CAST_array(c), - CAST_bitset(result)); + array_container_negation(CAST_array(c), CAST_bitset(result)); array_container_free(CAST_array(c)); return result; case RUN_CONTAINER_TYPE: *result_type = - run_container_negation_inplace(CAST_run(c), &result); + (uint8_t)run_container_negation_inplace(CAST_run(c), &result); return result; default: @@ -6055,31 +6033,28 @@ static inline container_t *container_inot( return NULL; } -static inline container_t *container_inot_range( - container_t *c, uint8_t type, - uint32_t range_start, uint32_t range_end, - uint8_t *result_type -){ +static inline container_t *container_inot_range(container_t *c, uint8_t type, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { c = get_writable_copy_if_shared(c, &type); container_t *result = NULL; switch (type) { case BITSET_CONTAINER_TYPE: - *result_type = - bitset_container_negation_range_inplace( - CAST_bitset(c), range_start, range_end, &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + *result_type = bitset_container_negation_range_inplace( + CAST_bitset(c), range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case ARRAY_CONTAINER_TYPE: - *result_type = - array_container_negation_range_inplace( - CAST_array(c), range_start, range_end, &result) - ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + *result_type = array_container_negation_range_inplace( + CAST_array(c), range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; return result; case RUN_CONTAINER_TYPE: - *result_type = run_container_negation_range_inplace( - CAST_run(c), range_start, range_end, &result); + *result_type = (uint8_t)run_container_negation_range_inplace( + CAST_run(c), range_start, range_end, &result); return result; default: @@ -6099,22 +6074,20 @@ static inline container_t *container_inot_range( * accordingly. * Otherwise, it returns false and update start_rank. */ -static inline bool container_select( - const container_t *c, uint8_t type, - uint32_t *start_rank, uint32_t rank, - uint32_t *element -){ +static inline bool container_select(const container_t *c, uint8_t type, + uint32_t *start_rank, uint32_t rank, + uint32_t *element) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: - return bitset_container_select(const_CAST_bitset(c), - start_rank, rank, element); + return bitset_container_select(const_CAST_bitset(c), start_rank, + rank, element); case ARRAY_CONTAINER_TYPE: - return array_container_select(const_CAST_array(c), - start_rank, rank, element); + return array_container_select(const_CAST_array(c), start_rank, rank, + element); case RUN_CONTAINER_TYPE: - return run_container_select(const_CAST_run(c), - start_rank, rank, element); + return run_container_select(const_CAST_run(c), start_rank, rank, + element); default: assert(false); roaring_unreachable; @@ -6124,9 +6097,7 @@ static inline bool container_select( return false; } -static inline uint16_t container_maximum( - const container_t *c, uint8_t type -){ +static inline uint16_t container_maximum(const container_t *c, uint8_t type) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -6144,9 +6115,7 @@ static inline uint16_t container_maximum( return false; } -static inline uint16_t container_minimum( - const container_t *c, uint8_t type -){ +static inline uint16_t container_minimum(const container_t *c, uint8_t type) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -6165,10 +6134,8 @@ static inline uint16_t container_minimum( } // number of values smaller or equal to x -static inline int container_rank( - const container_t *c, uint8_t type, - uint16_t x -){ +static inline int container_rank(const container_t *c, uint8_t type, + uint16_t x) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -6186,9 +6153,34 @@ static inline int container_rank( return false; } +// bulk version of container_rank(); return number of consumed elements +static inline uint32_t container_rank_many(const container_t *c, uint8_t type, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_rank_many(const_CAST_bitset(c), start_rank, + begin, end, ans); + case ARRAY_CONTAINER_TYPE: + return array_container_rank_many(const_CAST_array(c), start_rank, + begin, end, ans); + case RUN_CONTAINER_TYPE: + return run_container_rank_many(const_CAST_run(c), start_rank, begin, + end, ans); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return 0; +} + // return the index of x, if not exsist return -1 static inline int container_get_index(const container_t *c, uint8_t type, - uint16_t x) { + uint16_t x) { c = container_unwrap_shared(c, &type); switch (type) { case BITSET_CONTAINER_TYPE: @@ -6214,11 +6206,9 @@ static inline int container_get_index(const container_t *c, uint8_t type, * The type of the first container may change. Returns the modified * (and possibly new) container. */ -static inline container_t *container_add_range( - container_t *c, uint8_t type, - uint32_t min, uint32_t max, - uint8_t *result_type -){ +static inline container_t *container_add_range(container_t *c, uint8_t type, + uint32_t min, uint32_t max, + uint8_t *result_type) { // NB: when selecting new container type, we perform only inexpensive checks switch (type) { case BITSET_CONTAINER_TYPE: { @@ -6227,8 +6217,8 @@ static inline container_t *container_add_range( int32_t union_cardinality = 0; union_cardinality += bitset->cardinality; union_cardinality += max - min + 1; - union_cardinality -= bitset_lenrange_cardinality(bitset->words, - min, max-min); + union_cardinality -= + bitset_lenrange_cardinality(bitset->words, min, max - min); if (union_cardinality == INT32_C(0x10000)) { *result_type = RUN_CONTAINER_TYPE; @@ -6243,16 +6233,21 @@ static inline container_t *container_add_range( case ARRAY_CONTAINER_TYPE: { array_container_t *array = CAST_array(c); - int32_t nvals_greater = count_greater(array->array, array->cardinality, max); - int32_t nvals_less = count_less(array->array, array->cardinality - nvals_greater, min); - int32_t union_cardinality = nvals_less + (max - min + 1) + nvals_greater; + int32_t nvals_greater = + count_greater(array->array, array->cardinality, (uint16_t)max); + int32_t nvals_less = + count_less(array->array, array->cardinality - nvals_greater, + (uint16_t)min); + int32_t union_cardinality = + nvals_less + (max - min + 1) + nvals_greater; if (union_cardinality == INT32_C(0x10000)) { *result_type = RUN_CONTAINER_TYPE; return run_container_create_range(0, INT32_C(0x10000)); } else if (union_cardinality <= DEFAULT_MAX_SIZE) { *result_type = ARRAY_CONTAINER_TYPE; - array_container_add_range_nvals(array, min, max, nvals_less, nvals_greater); + array_container_add_range_nvals(array, min, max, nvals_less, + nvals_greater); return array; } else { *result_type = BITSET_CONTAINER_TYPE; @@ -6265,14 +6260,19 @@ static inline container_t *container_add_range( case RUN_CONTAINER_TYPE: { run_container_t *run = CAST_run(c); - int32_t nruns_greater = rle16_count_greater(run->runs, run->n_runs, max); - int32_t nruns_less = rle16_count_less(run->runs, run->n_runs - nruns_greater, min); + int32_t nruns_greater = + rle16_count_greater(run->runs, run->n_runs, (uint16_t)max); + int32_t nruns_less = rle16_count_less( + run->runs, run->n_runs - nruns_greater, (uint16_t)min); - int32_t run_size_bytes = (nruns_less + 1 + nruns_greater) * sizeof(rle16_t); - int32_t bitset_size_bytes = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + int32_t run_size_bytes = + (nruns_less + 1 + nruns_greater) * sizeof(rle16_t); + int32_t bitset_size_bytes = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); if (run_size_bytes <= bitset_size_bytes) { - run_container_add_range_nruns(run, min, max, nruns_less, nruns_greater); + run_container_add_range_nruns(run, min, max, nruns_less, + nruns_greater); *result_type = RUN_CONTAINER_TYPE; return run; } else { @@ -6292,30 +6292,30 @@ static inline container_t *container_add_range( * - pointer to a newly-allocated container (if it is more efficient) * * If the returned pointer is different from $container, then a new container - * has been created and the caller is responsible for freeing the original container. + * has been created and the caller is responsible for freeing the original + * container. */ -static inline container_t *container_remove_range( - container_t *c, uint8_t type, - uint32_t min, uint32_t max, - uint8_t *result_type -){ - switch (type) { +static inline container_t *container_remove_range(container_t *c, uint8_t type, + uint32_t min, uint32_t max, + uint8_t *result_type) { + switch (type) { case BITSET_CONTAINER_TYPE: { bitset_container_t *bitset = CAST_bitset(c); - int32_t result_cardinality = bitset->cardinality - - bitset_lenrange_cardinality(bitset->words, min, max-min); + int32_t result_cardinality = + bitset->cardinality - + bitset_lenrange_cardinality(bitset->words, min, max - min); if (result_cardinality == 0) { return NULL; } else if (result_cardinality <= DEFAULT_MAX_SIZE) { *result_type = ARRAY_CONTAINER_TYPE; - bitset_reset_range(bitset->words, min, max+1); + bitset_reset_range(bitset->words, min, max + 1); bitset->cardinality = result_cardinality; return array_container_from_bitset(bitset); } else { *result_type = BITSET_CONTAINER_TYPE; - bitset_reset_range(bitset->words, min, max+1); + bitset_reset_range(bitset->words, min, max + 1); bitset->cardinality = result_cardinality; return bitset; } @@ -6323,16 +6323,19 @@ static inline container_t *container_remove_range( case ARRAY_CONTAINER_TYPE: { array_container_t *array = CAST_array(c); - int32_t nvals_greater = count_greater(array->array, array->cardinality, max); - int32_t nvals_less = count_less(array->array, array->cardinality - nvals_greater, min); + int32_t nvals_greater = + count_greater(array->array, array->cardinality, (uint16_t)max); + int32_t nvals_less = + count_less(array->array, array->cardinality - nvals_greater, + (uint16_t)min); int32_t result_cardinality = nvals_less + nvals_greater; if (result_cardinality == 0) { return NULL; } else { *result_type = ARRAY_CONTAINER_TYPE; - array_container_remove_range(array, nvals_less, - array->cardinality - result_cardinality); + array_container_remove_range( + array, nvals_less, array->cardinality - result_cardinality); return array; } } @@ -6342,7 +6345,8 @@ static inline container_t *container_remove_range( if (run->n_runs == 0) { return NULL; } - if (min <= run_container_minimum(run) && max >= run_container_maximum(run)) { + if (min <= run_container_minimum(run) && + max >= run_container_maximum(run)) { return NULL; } @@ -6351,11 +6355,78 @@ static inline container_t *container_remove_range( } default: roaring_unreachable; - } + } } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +using api::roaring_container_iterator_t; +#endif + +/** + * Initializes the iterator at the first entry in the container. + */ +roaring_container_iterator_t container_init_iterator(const container_t *c, + uint8_t typecode, + uint16_t *value); + +/** + * Initializes the iterator at the last entry in the container. + */ +roaring_container_iterator_t container_init_iterator_last(const container_t *c, + uint8_t typecode, + uint16_t *value); + +/** + * Moves the iterator to the next entry. Returns true and sets `value` if a + * value is present. + */ +bool container_iterator_next(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, uint16_t *value); + +/** + * Moves the iterator to the previous entry. Returns true and sets `value` if a + * value is present. + */ +bool container_iterator_prev(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, uint16_t *value); + +/** + * Moves the iterator to the smallest entry that is greater than or equal to + * `val`. Returns true and sets `value_out` if a value is present. `value_out` + * should be initialized to a value. + */ +bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value_out, uint16_t val); + +/** + * Reads up to `count` entries from the container, and writes them into `buf` + * as `high16 | entry`. Returns true and sets `value_out` if a value is present + * after reading the entries. Sets `consumed` to the number of values read. + * `count` should be greater than zero. + */ +bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +/** + * Reads up to `count` entries from the container, and writes them into `buf` + * as `high48 | entry`. Returns true and sets `value_out` if a value is present + * after reading the entries. Sets `consumed` to the number of values read. + * `count` should be greater than zero. + */ +bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { #endif #endif @@ -6370,7 +6441,8 @@ static inline container_t *container_remove_range( #ifdef __cplusplus -extern "C" { namespace roaring { +extern "C" { +namespace roaring { // Note: in pure C++ code, you should avoid putting `using` in header files using api::roaring_array_t; @@ -6444,9 +6516,8 @@ inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x) { /** * Retrieves the container at index i, filling in the typecode */ -inline container_t *ra_get_container_at_index( - const roaring_array_t *ra, uint16_t i, uint8_t *typecode -){ +inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, uint8_t *typecode) { *typecode = ra->typecodes[i]; return ra->containers[i]; } @@ -6461,16 +6532,14 @@ inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, uint16_t i) { /** * Add a new key-value pair at index i */ -void ra_insert_new_key_value_at( - roaring_array_t *ra, int32_t i, uint16_t key, - container_t *c, uint8_t typecode); +void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, + container_t *c, uint8_t typecode); /** * Append a new key-value pair */ -void ra_append( - roaring_array_t *ra, uint16_t key, - container_t *c, uint8_t typecode); +void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, + uint8_t typecode); /** * Append a new key-value pair to ra, cloning (in COW sense) a value from sa @@ -6520,16 +6589,15 @@ void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, * Set the container at the corresponding index using the specified * typecode. */ -inline void ra_set_container_at_index( - const roaring_array_t *ra, int32_t i, - container_t *c, uint8_t typecode -){ +inline void ra_set_container_at_index(const roaring_array_t *ra, int32_t i, + container_t *c, uint8_t typecode) { assert(i < ra->size); ra->containers[i] = c; ra->typecodes[i] = typecode; } -container_t *ra_get_container(roaring_array_t *ra, uint16_t x, uint8_t *typecode); +container_t *ra_get_container(roaring_array_t *ra, uint16_t x, + uint8_t *typecode); /** * If needed, increase the capacity of the array so that it can fit k values @@ -6549,10 +6617,10 @@ int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos); void ra_downsize(roaring_array_t *ra, int32_t new_length); -inline void ra_replace_key_and_container_at_index( - roaring_array_t *ra, int32_t i, uint16_t key, - container_t *c, uint8_t typecode -){ +inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, + int32_t i, uint16_t key, + container_t *c, + uint8_t typecode) { assert(i < ra->size); ra->keys[i] = key; @@ -6563,7 +6631,8 @@ inline void ra_replace_key_and_container_at_index( // write set bits to an array void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans); -bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, size_t limit, uint32_t *ans); +bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, + size_t limit, uint32_t *ans); /** * write a bitmap to a buffer. This is meant to be compatible with @@ -6578,10 +6647,11 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf); * with the Java and Go versions. * maxbytes indicates how many bytes available from buf. * When the function returns true, roaring_array_t is populated with the data - * and *readbytes indicates how many bytes were read. In all cases, if the function - * returns true, then maxbytes >= *readbytes. + * and *readbytes indicates how many bytes were read. In all cases, if the + * function returns true, then maxbytes >= *readbytes. */ -bool ra_portable_deserialize(roaring_array_t *ra, const char *buf, const size_t maxbytes, size_t * readbytes); +bool ra_portable_deserialize(roaring_array_t *ra, const char *buf, + const size_t maxbytes, size_t *readbytes); /** * Quickly checks whether there is a serialized bitmap at the pointer, @@ -6618,8 +6688,8 @@ uint32_t ra_portable_header_size(const roaring_array_t *ra); static inline void ra_unshare_container_at_index(roaring_array_t *ra, uint16_t i) { assert(i < ra->size); - ra->containers[i] = get_writable_copy_if_shared(ra->containers[i], - &ra->typecodes[i]); + ra->containers[i] = + get_writable_copy_if_shared(ra->containers[i], &ra->typecodes[i]); } /** @@ -6627,10 +6697,9 @@ static inline void ra_unshare_container_at_index(roaring_array_t *ra, */ void ra_remove_at_index(roaring_array_t *ra, int32_t i); - /** -* clears all containers, sets the size at 0 and shrinks the memory usage. -*/ + * clears all containers, sets the size at 0 and shrinks the memory usage. + */ void ra_reset(roaring_array_t *ra); /** @@ -6661,11 +6730,206 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); #ifdef __cplusplus } // namespace internal -} } // extern "C" { namespace roaring { +} +} // extern "C" { namespace roaring { #endif #endif /* end file include/roaring/roaring_array.h */ +/* begin file include/roaring/art/art.h */ +#ifndef ART_ART_H +#define ART_ART_H + +#include +#include +#include + +/* + * This file contains an implementation of an Adaptive Radix Tree as described + * in https://db.in.tum.de/~leis/papers/ART.pdf. + * + * The ART contains the keys in _byte lexographical_ order. + * + * Other features: + * * Fixed 48 bit key length: all keys are assumed to be be 48 bits in size. + * This allows us to put the key and key prefixes directly in nodes, reducing + * indirection at no additional memory overhead. + * * Key compression: the only inner nodes created are at points where key + * chunks _differ_. This means that if there are two entries with different + * high 48 bits, then there is only one inner node containing the common key + * prefix, and two leaves. + * * Intrusive leaves: the leaf struct is included in user values. This removes + * a layer of indirection. + */ + +// Fixed length of keys in the ART. All keys are assumed to be of this length. +#define ART_KEY_BYTES 6 + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +typedef uint8_t art_key_chunk_t; +typedef struct art_node_s art_node_t; + +/** + * Wrapper to allow an empty tree. + */ +typedef struct art_s { + art_node_t *root; +} art_t; + +/** + * Values inserted into the tree have to be cast-able to art_val_t. This + * improves performance by reducing indirection. + * + * NOTE: Value pointers must be unique! This is because each value struct + * contains the key corresponding to the value. + */ +typedef struct art_val_s { + art_key_chunk_t key[ART_KEY_BYTES]; +} art_val_t; + +/** + * Compares two keys, returns their relative order: + * * Key 1 < key 2: returns a negative value + * * Key 1 == key 2: returns 0 + * * Key 1 > key 2: returns a positive value + */ +int art_compare_keys(const art_key_chunk_t key1[], + const art_key_chunk_t key2[]); + +/** + * Inserts the given key and value. + */ +void art_insert(art_t *art, const art_key_chunk_t *key, art_val_t *val); + +/** + * Returns the value erased, NULL if not found. + */ +art_val_t *art_erase(art_t *art, const art_key_chunk_t *key); + +/** + * Returns the value associated with the given key, NULL if not found. + */ +art_val_t *art_find(const art_t *art, const art_key_chunk_t *key); + +/** + * Returns true if the ART is empty. + */ +bool art_is_empty(const art_t *art); + +/** + * Frees the nodes of the ART except the values, which the user is expected to + * free. + */ +void art_free(art_t *art); + +/** + * Returns the size in bytes of the ART. Includes size of pointers to values, + * but not the values themselves. + */ +size_t art_size_in_bytes(const art_t *art); + +/** + * Prints the ART using printf, useful for debugging. + */ +void art_printf(const art_t *art); + +/** + * Callback for validating the value stored in a leaf. + * + * Should return true if the value is valid, false otherwise + * If false is returned, `*reason` should be set to a static string describing + * the reason for the failure. + */ +typedef bool (*art_validate_cb_t)(const art_val_t *val, const char **reason); + +/** + * Validate the ART tree, ensuring it is internally consistent. + */ +bool art_internal_validate(const art_t *art, const char **reason, + art_validate_cb_t validate_cb); + +/** + * ART-internal iterator bookkeeping. Users should treat this as an opaque type. + */ +typedef struct art_iterator_frame_s { + art_node_t *node; + uint8_t index_in_node; +} art_iterator_frame_t; + +/** + * Users should only access `key` and `value` in iterators. The iterator is + * valid when `value != NULL`. + */ +typedef struct art_iterator_s { + art_key_chunk_t key[ART_KEY_BYTES]; + art_val_t *value; + + uint8_t depth; // Key depth + uint8_t frame; // Node depth + + // State for each node in the ART the iterator has travelled from the root. + // This is `ART_KEY_BYTES + 1` because it includes state for the leaf too. + art_iterator_frame_t frames[ART_KEY_BYTES + 1]; +} art_iterator_t; + +/** + * Creates an iterator initialzed to the first or last entry in the ART, + * depending on `first`. The iterator is not valid if there are no entries in + * the ART. + */ +art_iterator_t art_init_iterator(const art_t *art, bool first); + +/** + * Returns an initialized iterator positioned at a key equal to or greater than + * the given key, if it exists. + */ +art_iterator_t art_lower_bound(const art_t *art, const art_key_chunk_t *key); + +/** + * Returns an initialized iterator positioned at a key greater than the given + * key, if it exists. + */ +art_iterator_t art_upper_bound(const art_t *art, const art_key_chunk_t *key); + +/** + * The following iterator movement functions return true if a new entry was + * encountered. + */ +bool art_iterator_move(art_iterator_t *iterator, bool forward); +bool art_iterator_next(art_iterator_t *iterator); +bool art_iterator_prev(art_iterator_t *iterator); + +/** + * Moves the iterator forward to a key equal to or greater than the given key. + */ +bool art_iterator_lower_bound(art_iterator_t *iterator, + const art_key_chunk_t *key); + +/** + * Insert the value and positions the iterator at the key. + */ +void art_iterator_insert(art_t *art, art_iterator_t *iterator, + const art_key_chunk_t *key, art_val_t *val); + +/** + * Erase the value pointed at by the iterator. Moves the iterator to the next + * leaf. Returns the value erased or NULL if nothing was erased. + */ +art_val_t *art_iterator_erase(art_t *art, art_iterator_t *iterator); + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace internal +#endif + +#endif +/* end file include/roaring/art/art.h */ /* begin file src/array_util.c */ #include #include @@ -6678,12 +6942,19 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 #endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif #ifdef __cplusplus using namespace ::roaring::internal; -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif extern inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, @@ -7119,18 +7390,21 @@ int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, } ALLOW_UNALIGNED -int array_container_to_uint32_array_vector16(void *vout, const uint16_t* array, size_t cardinality, - uint32_t base) { +int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, + size_t cardinality, + uint32_t base) { int outpos = 0; uint32_t *out = (uint32_t *)vout; size_t i = 0; - for ( ;i + sizeof(__m128i)/sizeof(uint16_t) <= cardinality; i += sizeof(__m128i)/sizeof(uint16_t)) { - __m128i vinput = _mm_loadu_si128((const __m128i*) (array + i)); - __m256i voutput = _mm256_add_epi32(_mm256_cvtepu16_epi32(vinput), _mm256_set1_epi32(base)); - _mm256_storeu_si256((__m256i*)(out + outpos), voutput); - outpos += sizeof(__m256i)/sizeof(uint32_t); - } - for ( ; i < cardinality; ++i) { + for (; i + sizeof(__m128i) / sizeof(uint16_t) <= cardinality; + i += sizeof(__m128i) / sizeof(uint16_t)) { + __m128i vinput = _mm_loadu_si128((const __m128i *)(array + i)); + __m256i voutput = _mm256_add_epi32(_mm256_cvtepu16_epi32(vinput), + _mm256_set1_epi32(base)); + _mm256_storeu_si256((__m256i *)(out + outpos), voutput); + outpos += sizeof(__m256i) / sizeof(uint32_t); + } + for (; i < cardinality; ++i) { const uint32_t val = base + array[i]; memcpy(out + outpos, &val, sizeof(uint32_t)); // should be compiled as a MOV on x64 @@ -7140,7 +7414,7 @@ int array_container_to_uint32_array_vector16(void *vout, const uint16_t* array, } int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b) { + const uint16_t *__restrict__ B, size_t s_b) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -7159,7 +7433,7 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, const int r = _mm_extract_epi32(res_v, 0); __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); __m128i p = _mm_shuffle_epi8(v_a, sm16); - _mm_storeu_si128((__m128i*)&((uint16_t*)tmp)[tmp_count], p); + _mm_storeu_si128((__m128i *)&((uint16_t *)tmp)[tmp_count], p); tmp_count += _mm_popcnt_u32(r); const uint16_t a_max = A[i_a + vectorlength - 1]; const uint16_t b_max = B[i_b + vectorlength - 1]; @@ -7167,7 +7441,7 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, _mm_storeu_si128((__m128i *)&A[count], tmp[0]); _mm_storeu_si128(tmp, _mm_setzero_si128()); count += tmp_count; - tmp_count = 0; + tmp_count = 0; i_a += vectorlength; if (i_a == st_a) break; v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); @@ -7184,9 +7458,10 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, v_b, v_a, _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); const int r = _mm_extract_epi32(res_v, 0); - __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); + __m128i sm16 = + _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); __m128i p = _mm_shuffle_epi8(v_a, sm16); - _mm_storeu_si128((__m128i*)&((uint16_t*)tmp)[tmp_count], p); + _mm_storeu_si128((__m128i *)&((uint16_t *)tmp)[tmp_count], p); tmp_count += _mm_popcnt_u32(r); const uint16_t a_max = A[i_a + vectorlength - 1]; const uint16_t b_max = B[i_b + vectorlength - 1]; @@ -7194,7 +7469,7 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, _mm_storeu_si128((__m128i *)&A[count], tmp[0]); _mm_storeu_si128(tmp, _mm_setzero_si128()); count += tmp_count; - tmp_count = 0; + tmp_count = 0; i_a += vectorlength; if (i_a == st_a) break; v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); @@ -7208,7 +7483,7 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, } // tmp_count <= 8, so this does not affect efficiency so much for (size_t i = 0; i < tmp_count; i++) { - A[count] = ((uint16_t*)tmp)[i]; + A[count] = ((uint16_t *)tmp)[i]; count++; } i_a += tmp_count; // We can at least jump pass $tmp_count elements in A @@ -7351,15 +7626,15 @@ int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, // spotted in B, these don't get written out. __m128i runningmask_a_found_in_b = _mm_setzero_si128(); /**** - * start of the main vectorized loop - *****/ + * start of the main vectorized loop + *****/ while (true) { // afoundinb will contain a mask indicate for each entry in A // whether it is seen // in B - const __m128i a_found_in_b = - _mm_cmpistrm(v_b, v_a, _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | - _SIDD_BIT_MASK); + const __m128i a_found_in_b = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); runningmask_a_found_in_b = _mm_or_si128(runningmask_a_found_in_b, a_found_in_b); // we always compare the last values of A and B @@ -7373,7 +7648,7 @@ int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, _mm_extract_epi32(runningmask_a_found_in_b, 0) ^ 0xFF; /*** next few lines are probably expensive *****/ __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + - bitmask_belongs_to_difference); + bitmask_belongs_to_difference); __m128i p = _mm_shuffle_epi8(v_a, sm16); _mm_storeu_si128((__m128i *)&C[count], p); // can overflow count += _mm_popcnt_u32(bitmask_belongs_to_difference); @@ -7400,15 +7675,15 @@ int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, memset(buffer, 0, 8 * sizeof(uint16_t)); memcpy(buffer, B + i_b, (s_b - i_b) * sizeof(uint16_t)); v_b = _mm_lddqu_si128((__m128i *)buffer); - const __m128i a_found_in_b = - _mm_cmpistrm(v_b, v_a, _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | - _SIDD_BIT_MASK); + const __m128i a_found_in_b = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); runningmask_a_found_in_b = _mm_or_si128(runningmask_a_found_in_b, a_found_in_b); const int bitmask_belongs_to_difference = _mm_extract_epi32(runningmask_a_found_in_b, 0) ^ 0xFF; __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + - bitmask_belongs_to_difference); + bitmask_belongs_to_difference); __m128i p = _mm_shuffle_epi8(v_a, sm16); _mm_storeu_si128((__m128i *)&C[count], p); // can overflow count += _mm_popcnt_u32(bitmask_belongs_to_difference); @@ -7432,15 +7707,15 @@ int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, } } if (i_a < s_a) { - if(C == A) { - assert((size_t)count <= i_a); - if((size_t)count < i_a) { - memmove(C + count, A + i_a, sizeof(uint16_t) * (s_a - i_a)); - } + if (C == A) { + assert((size_t)count <= i_a); + if ((size_t)count < i_a) { + memmove(C + count, A + i_a, sizeof(uint16_t) * (s_a - i_a)); + } } else { - for(size_t i = 0; i < (s_a - i_a); i++) { + for (size_t i = 0; i < (s_a - i_a); i++) { C[count + i] = A[i + i_a]; - } + } } count += (int32_t)(s_a - i_a); } @@ -7449,62 +7724,58 @@ int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, CROARING_UNTARGET_AVX2 #endif // CROARING_IS_X64 - - /** -* Branchless binary search going after 4 values at once. -* Assumes that array is sorted. -* You have that array[*index1] >= target1, array[*index12] >= target2, ... -* except when *index1 = n, in which case you know that all values in array are -* smaller than target1, and so forth. -* It has logarithmic complexity. -*/ + * Branchless binary search going after 4 values at once. + * Assumes that array is sorted. + * You have that array[*index1] >= target1, array[*index12] >= target2, ... + * except when *index1 = n, in which case you know that all values in array are + * smaller than target1, and so forth. + * It has logarithmic complexity. + */ static void binarySearch4(const uint16_t *array, int32_t n, uint16_t target1, - uint16_t target2, uint16_t target3, uint16_t target4, - int32_t *index1, int32_t *index2, int32_t *index3, - int32_t *index4) { - const uint16_t *base1 = array; - const uint16_t *base2 = array; - const uint16_t *base3 = array; - const uint16_t *base4 = array; - if (n == 0) - return; - while (n > 1) { - int32_t half = n >> 1; - base1 = (base1[half] < target1) ? &base1[half] : base1; - base2 = (base2[half] < target2) ? &base2[half] : base2; - base3 = (base3[half] < target3) ? &base3[half] : base3; - base4 = (base4[half] < target4) ? &base4[half] : base4; - n -= half; - } - *index1 = (int32_t)((*base1 < target1) + base1 - array); - *index2 = (int32_t)((*base2 < target2) + base2 - array); - *index3 = (int32_t)((*base3 < target3) + base3 - array); - *index4 = (int32_t)((*base4 < target4) + base4 - array); + uint16_t target2, uint16_t target3, uint16_t target4, + int32_t *index1, int32_t *index2, int32_t *index3, + int32_t *index4) { + const uint16_t *base1 = array; + const uint16_t *base2 = array; + const uint16_t *base3 = array; + const uint16_t *base4 = array; + if (n == 0) return; + while (n > 1) { + int32_t half = n >> 1; + base1 = (base1[half] < target1) ? &base1[half] : base1; + base2 = (base2[half] < target2) ? &base2[half] : base2; + base3 = (base3[half] < target3) ? &base3[half] : base3; + base4 = (base4[half] < target4) ? &base4[half] : base4; + n -= half; + } + *index1 = (int32_t)((*base1 < target1) + base1 - array); + *index2 = (int32_t)((*base2 < target2) + base2 - array); + *index3 = (int32_t)((*base3 < target3) + base3 - array); + *index4 = (int32_t)((*base4 < target4) + base4 - array); } /** -* Branchless binary search going after 2 values at once. -* Assumes that array is sorted. -* You have that array[*index1] >= target1, array[*index12] >= target2. -* except when *index1 = n, in which case you know that all values in array are -* smaller than target1, and so forth. -* It has logarithmic complexity. -*/ + * Branchless binary search going after 2 values at once. + * Assumes that array is sorted. + * You have that array[*index1] >= target1, array[*index12] >= target2. + * except when *index1 = n, in which case you know that all values in array are + * smaller than target1, and so forth. + * It has logarithmic complexity. + */ static void binarySearch2(const uint16_t *array, int32_t n, uint16_t target1, - uint16_t target2, int32_t *index1, int32_t *index2) { - const uint16_t *base1 = array; - const uint16_t *base2 = array; - if (n == 0) - return; - while (n > 1) { - int32_t half = n >> 1; - base1 = (base1[half] < target1) ? &base1[half] : base1; - base2 = (base2[half] < target2) ? &base2[half] : base2; - n -= half; - } - *index1 = (int32_t)((*base1 < target1) + base1 - array); - *index2 = (int32_t)((*base2 < target2) + base2 - array); + uint16_t target2, int32_t *index1, int32_t *index2) { + const uint16_t *base1 = array; + const uint16_t *base2 = array; + if (n == 0) return; + while (n > 1) { + int32_t half = n >> 1; + base1 = (base1[half] < target1) ? &base1[half] : base1; + base2 = (base2[half] < target2) ? &base2[half] : base2; + n -= half; + } + *index1 = (int32_t)((*base1 < target1) + base1 - array); + *index2 = (int32_t)((*base2 < target2) + base2 - array); } /* Computes the intersection between one small and one large set of uint16_t. @@ -7514,61 +7785,60 @@ static void binarySearch2(const uint16_t *array, int32_t n, uint16_t target1, * galloping search in some instances. */ int32_t intersect_skewed_uint16(const uint16_t *small, size_t size_s, - const uint16_t *large, size_t size_l, - uint16_t *buffer) { - size_t pos = 0, idx_l = 0, idx_s = 0; + const uint16_t *large, size_t size_l, + uint16_t *buffer) { + size_t pos = 0, idx_l = 0, idx_s = 0; - if (0 == size_s) { - return 0; - } - int32_t index1 = 0, index2 = 0, index3 = 0, index4 = 0; - while ((idx_s + 4 <= size_s) && (idx_l < size_l)) { - uint16_t target1 = small[idx_s]; - uint16_t target2 = small[idx_s + 1]; - uint16_t target3 = small[idx_s + 2]; - uint16_t target4 = small[idx_s + 3]; - binarySearch4(large + idx_l, (int32_t)(size_l - idx_l), target1, target2, target3, - target4, &index1, &index2, &index3, &index4); - if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { - buffer[pos++] = target1; - } - if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { - buffer[pos++] = target2; - } - if ((index3 + idx_l < size_l) && (large[idx_l + index3] == target3)) { - buffer[pos++] = target3; - } - if ((index4 + idx_l < size_l) && (large[idx_l + index4] == target4)) { - buffer[pos++] = target4; - } - idx_s += 4; - idx_l += index4; - } - if ((idx_s + 2 <= size_s) && (idx_l < size_l)) { - uint16_t target1 = small[idx_s]; - uint16_t target2 = small[idx_s + 1]; - binarySearch2(large + idx_l, (int32_t)(size_l - idx_l), target1, target2, &index1, - &index2); - if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { - buffer[pos++] = target1; - } - if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { - buffer[pos++] = target2; - } - idx_s += 2; - idx_l += index2; - } - if ((idx_s < size_s) && (idx_l < size_l)) { - uint16_t val_s = small[idx_s]; - int32_t index = binarySearch(large + idx_l, (int32_t)(size_l - idx_l), val_s); - if (index >= 0) - buffer[pos++] = val_s; - } - return (int32_t)pos; + if (0 == size_s) { + return 0; + } + int32_t index1 = 0, index2 = 0, index3 = 0, index4 = 0; + while ((idx_s + 4 <= size_s) && (idx_l < size_l)) { + uint16_t target1 = small[idx_s]; + uint16_t target2 = small[idx_s + 1]; + uint16_t target3 = small[idx_s + 2]; + uint16_t target4 = small[idx_s + 3]; + binarySearch4(large + idx_l, (int32_t)(size_l - idx_l), target1, + target2, target3, target4, &index1, &index2, &index3, + &index4); + if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { + buffer[pos++] = target1; + } + if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { + buffer[pos++] = target2; + } + if ((index3 + idx_l < size_l) && (large[idx_l + index3] == target3)) { + buffer[pos++] = target3; + } + if ((index4 + idx_l < size_l) && (large[idx_l + index4] == target4)) { + buffer[pos++] = target4; + } + idx_s += 4; + idx_l += index4; + } + if ((idx_s + 2 <= size_s) && (idx_l < size_l)) { + uint16_t target1 = small[idx_s]; + uint16_t target2 = small[idx_s + 1]; + binarySearch2(large + idx_l, (int32_t)(size_l - idx_l), target1, + target2, &index1, &index2); + if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { + buffer[pos++] = target1; + } + if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { + buffer[pos++] = target2; + } + idx_s += 2; + idx_l += index2; + } + if ((idx_s < size_s) && (idx_l < size_l)) { + uint16_t val_s = small[idx_s]; + int32_t index = + binarySearch(large + idx_l, (int32_t)(size_l - idx_l), val_s); + if (index >= 0) buffer[pos++] = val_s; + } + return (int32_t)pos; } - - // TODO: this could be accelerated, possibly, by using binarySearch4 as above. int32_t intersect_skewed_uint16_cardinality(const uint16_t *small, size_t size_s, @@ -7606,7 +7876,7 @@ int32_t intersect_skewed_uint16_cardinality(const uint16_t *small, } bool intersect_skewed_uint16_nonempty(const uint16_t *small, size_t size_s, - const uint16_t *large, size_t size_l) { + const uint16_t *large, size_t size_l) { size_t idx_l = 0, idx_s = 0; if (0 == size_s) { @@ -7657,7 +7927,7 @@ int32_t intersect_uint16(const uint16_t *A, const size_t lenA, goto SKIP_FIRST_COMPARE; } } - return (int32_t)(out - initout); // NOTREACHED + // return (int32_t)(out - initout); // NOTREACHED } int32_t intersect_uint16_cardinality(const uint16_t *A, const size_t lenA, @@ -7682,12 +7952,11 @@ int32_t intersect_uint16_cardinality(const uint16_t *A, const size_t lenA, goto SKIP_FIRST_COMPARE; } } - return answer; // NOTREACHED + // return answer; // NOTREACHED } - bool intersect_uint16_nonempty(const uint16_t *A, const size_t lenA, - const uint16_t *B, const size_t lenB) { + const uint16_t *B, const size_t lenB) { if (lenA == 0 || lenB == 0) return 0; const uint16_t *endA = A + lenA; const uint16_t *endB = B + lenB; @@ -7709,8 +7978,6 @@ bool intersect_uint16_nonempty(const uint16_t *A, const size_t lenA, return false; // NOTREACHED } - - /** * Generic intersection function. */ @@ -7737,7 +8004,7 @@ size_t intersection_uint32(const uint32_t *A, const size_t lenA, goto SKIP_FIRST_COMPARE; } } - return (out - initout); // NOTREACHED + // return (out - initout); // NOTREACHED } size_t intersection_uint32_card(const uint32_t *A, const size_t lenA, @@ -7762,7 +8029,7 @@ size_t intersection_uint32_card(const uint32_t *A, const size_t lenA, goto SKIP_FIRST_COMPARE; } } - return card; // NOTREACHED + // return card; // NOTREACHED } // can one vectorize the computation of the union? (Update: Yes! See @@ -8508,8 +8775,8 @@ uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, // last value of vecMax, // we store to "buffer" int leftoversize = store_unique_xor(laststore, vecMax, buffer); - uint16_t vec7 = _mm_extract_epi16(vecMax, 7); - uint16_t vec6 = _mm_extract_epi16(vecMax, 6); + uint16_t vec7 = (uint16_t)_mm_extract_epi16(vecMax, 7); + uint16_t vec6 = (uint16_t)_mm_extract_epi16(vecMax, 6); if (vec7 != vec6) buffer[leftoversize++] = vec7; if (pos1 == len1) { memcpy(buffer + leftoversize, array1 + 8 * pos1, @@ -8642,38 +8909,33 @@ size_t union_uint32_card(const uint32_t *set_1, size_t size_1, return pos; } - - -size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, - size_t size_2, uint16_t *buffer) { +size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, + const uint16_t *set_2, size_t size_2, + uint16_t *buffer) { #if CROARING_IS_X64 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { // compute union with smallest array first - if (size_1 < size_2) { - return union_vector16(set_1, (uint32_t)size_1, - set_2, (uint32_t)size_2, buffer); - } else { - return union_vector16(set_2, (uint32_t)size_2, - set_1, (uint32_t)size_1, buffer); - } + if (size_1 < size_2) { + return union_vector16(set_1, (uint32_t)size_1, set_2, + (uint32_t)size_2, buffer); + } else { + return union_vector16(set_2, (uint32_t)size_2, set_1, + (uint32_t)size_1, buffer); + } } else { - // compute union with smallest array first - if (size_1 < size_2) { - return union_uint16( - set_1, size_1, set_2, size_2, buffer); - } else { - return union_uint16( - set_2, size_2, set_1, size_1, buffer); - } + // compute union with smallest array first + if (size_1 < size_2) { + return union_uint16(set_1, size_1, set_2, size_2, buffer); + } else { + return union_uint16(set_2, size_2, set_1, size_1, buffer); + } } #else // compute union with smallest array first if (size_1 < size_2) { - return union_uint16( - set_1, size_1, set_2, size_2, buffer); + return union_uint16(set_1, size_1, set_2, size_2, buffer); } else { - return union_uint16( - set_2, size_2, set_1, size_1, buffer); + return union_uint16(set_2, size_2, set_1, size_1, buffer); } #endif } @@ -8687,25 +8949,24 @@ static inline bool _avx512_memequals(const void *s1, const void *s2, size_t n) { const uint8_t *end8 = ptr1 + ((n >> 3) << 3); const uint8_t *end32 = ptr1 + ((n >> 5) << 5); const uint8_t *end64 = ptr1 + ((n >> 6) << 6); - - while (ptr1 < end64){ - __m512i r1 = _mm512_loadu_si512((const __m512i*)ptr1); - __m512i r2 = _mm512_loadu_si512((const __m512i*)ptr2); + + while (ptr1 < end64) { + __m512i r1 = _mm512_loadu_si512((const __m512i *)ptr1); + __m512i r2 = _mm512_loadu_si512((const __m512i *)ptr2); uint64_t mask = _mm512_cmpeq_epi8_mask(r1, r2); - + if (mask != UINT64_MAX) { - return false; + return false; } ptr1 += 64; ptr2 += 64; - } while (ptr1 < end32) { - __m256i r1 = _mm256_loadu_si256((const __m256i*)ptr1); - __m256i r2 = _mm256_loadu_si256((const __m256i*)ptr2); + __m256i r1 = _mm256_loadu_si256((const __m256i *)ptr1); + __m256i r2 = _mm256_loadu_si256((const __m256i *)ptr2); int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(r1, r2)); if ((uint32_t)mask != UINT32_MAX) { return false; @@ -8715,9 +8976,9 @@ static inline bool _avx512_memequals(const void *s1, const void *s2, size_t n) { } while (ptr1 < end8) { - uint64_t v1, v2; - memcpy(&v1,ptr1,sizeof(uint64_t)); - memcpy(&v2,ptr2,sizeof(uint64_t)); + uint64_t v1, v2; + memcpy(&v1, ptr1, sizeof(uint64_t)); + memcpy(&v2, ptr2, sizeof(uint64_t)); if (v1 != v2) { return false; } @@ -8736,19 +8997,19 @@ static inline bool _avx512_memequals(const void *s1, const void *s2, size_t n) { return true; } CROARING_UNTARGET_AVX512 -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX2 static inline bool _avx2_memequals(const void *s1, const void *s2, size_t n) { const uint8_t *ptr1 = (const uint8_t *)s1; const uint8_t *ptr2 = (const uint8_t *)s2; const uint8_t *end1 = ptr1 + n; - const uint8_t *end8 = ptr1 + n/8*8; - const uint8_t *end32 = ptr1 + n/32*32; + const uint8_t *end8 = ptr1 + n / 8 * 8; + const uint8_t *end32 = ptr1 + n / 32 * 32; while (ptr1 < end32) { - __m256i r1 = _mm256_loadu_si256((const __m256i*)ptr1); - __m256i r2 = _mm256_loadu_si256((const __m256i*)ptr2); + __m256i r1 = _mm256_loadu_si256((const __m256i *)ptr1); + __m256i r2 = _mm256_loadu_si256((const __m256i *)ptr2); int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(r1, r2)); if ((uint32_t)mask != UINT32_MAX) { return false; @@ -8759,8 +9020,8 @@ static inline bool _avx2_memequals(const void *s1, const void *s2, size_t n) { while (ptr1 < end8) { uint64_t v1, v2; - memcpy(&v1,ptr1,sizeof(uint64_t)); - memcpy(&v2,ptr2,sizeof(uint64_t)); + memcpy(&v1, ptr1, sizeof(uint64_t)); + memcpy(&v2, ptr2, sizeof(uint64_t)); if (v1 != v2) { return false; } @@ -8788,37 +9049,38 @@ bool memequals(const void *s1, const void *s2, size_t n) { #if CROARING_IS_X64 int support = croaring_hardware_support(); #if CROARING_COMPILER_SUPPORTS_AVX512 - if( support & ROARING_SUPPORTS_AVX512 ) { - return _avx512_memequals(s1, s2, n); + if (support & ROARING_SUPPORTS_AVX512) { + return _avx512_memequals(s1, s2, n); } else -#endif // CROARING_COMPILER_SUPPORTS_AVX512 - if( support & ROARING_SUPPORTS_AVX2 ) { - return _avx2_memequals(s1, s2, n); - } else { - return memcmp(s1, s2, n) == 0; - } +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { + return _avx2_memequals(s1, s2, n); + } else { + return memcmp(s1, s2, n) == 0; + } #else return memcmp(s1, s2, n) == 0; #endif } - #if CROARING_IS_X64 #if CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX512 ALLOW_UNALIGNED -int avx512_array_container_to_uint32_array(void *vout, const uint16_t* array, size_t cardinality, - uint32_t base) { +int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base) { int outpos = 0; uint32_t *out = (uint32_t *)vout; size_t i = 0; - for ( ;i + sizeof(__m256i)/sizeof(uint16_t) <= cardinality; i += sizeof(__m256i)/sizeof(uint16_t)) { - __m256i vinput = _mm256_loadu_si256((const __m256i*) (array + i)); - __m512i voutput = _mm512_add_epi32(_mm512_cvtepu16_epi32(vinput), _mm512_set1_epi32(base)); - _mm512_storeu_si512((__m512i*)(out + outpos), voutput); - outpos += sizeof(__m512i)/sizeof(uint32_t); - } - for ( ; i < cardinality; ++i) { + for (; i + sizeof(__m256i) / sizeof(uint16_t) <= cardinality; + i += sizeof(__m256i) / sizeof(uint16_t)) { + __m256i vinput = _mm256_loadu_si256((const __m256i *)(array + i)); + __m512i voutput = _mm512_add_epi32(_mm512_cvtepu16_epi32(vinput), + _mm512_set1_epi32(base)); + _mm512_storeu_si512((__m512i *)(out + outpos), voutput); + outpos += sizeof(__m512i) / sizeof(uint32_t); + } + for (; i < cardinality; ++i) { const uint32_t val = base + array[i]; memcpy(out + outpos, &val, sizeof(uint32_t)); // should be compiled as a MOV on x64 @@ -8827,467 +9089,2392 @@ int avx512_array_container_to_uint32_array(void *vout, const uint16_t* array, si return outpos; } CROARING_UNTARGET_AVX512 -#endif // #if CROARING_COMPILER_SUPPORTS_AVX512 -#endif // #if CROARING_IS_X64 - +#endif // #if CROARING_COMPILER_SUPPORTS_AVX512 +#endif // #if CROARING_IS_X64 #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif -/* end file src/array_util.c */ -/* begin file src/bitset.c */ -#include -#include +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif/* end file src/array_util.c */ +/* begin file src/art/art.c */ +#include #include -#include #include +#define CROARING_ART_NODE4_TYPE 0 +#define CROARING_ART_NODE16_TYPE 1 +#define CROARING_ART_NODE48_TYPE 2 +#define CROARING_ART_NODE256_TYPE 3 +#define CROARING_ART_NUM_TYPES 4 + +// Node48 placeholder value to indicate no child is present at this key index. +#define CROARING_ART_NODE48_EMPTY_VAL 48 + +// We use the least significant bit of node pointers to indicate whether a node +// is a leaf or an inner node. This is never surfaced to the user. +// +// Using pointer tagging to indicate leaves not only saves a bit of memory by +// sparing the typecode, but also allows us to use an intrusive leaf struct. +// Using an intrusive leaf struct leaves leaf allocation up to the user. Upon +// deallocation of the ART, we know not to free the leaves without having to +// dereference the leaf pointers. +// +// All internal operations on leaves should use CROARING_CAST_LEAF before using +// the leaf. The only places that use CROARING_SET_LEAF are locations where a +// field is directly assigned to a leaf pointer. After using CROARING_SET_LEAF, +// the leaf should be treated as a node of unknown type. +#define CROARING_IS_LEAF(p) (((uintptr_t)(p) & 1)) +#define CROARING_SET_LEAF(p) ((art_node_t *)((uintptr_t)(p) | 1)) +#define CROARING_CAST_LEAF(p) ((art_leaf_t *)((void *)((uintptr_t)(p) & ~1))) + +#define CROARING_NODE48_AVAILABLE_CHILDREN_MASK ((UINT64_C(1) << 48) - 1) + #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif -extern inline void bitset_print(const bitset_t *b); -extern inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, - void *ptr); -extern inline size_t bitset_next_set_bits(const bitset_t *bitset, size_t *buffer, - size_t capacity, size_t *startfrom); -extern inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag); -extern inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i); -extern inline void bitset_set(bitset_t *bitset, size_t i); -extern inline bool bitset_get(const bitset_t *bitset, size_t i); -extern inline size_t bitset_size_in_words(const bitset_t *bitset); -extern inline size_t bitset_size_in_bits(const bitset_t *bitset); -extern inline size_t bitset_size_in_bytes(const bitset_t *bitset); +typedef uint8_t art_typecode_t; +// Aliasing with a "leaf" naming so that its purpose is clearer in the context +// of the trie internals. +typedef art_val_t art_leaf_t; -/* Create a new bitset. Return NULL in case of failure. */ -bitset_t *bitset_create(void) { - bitset_t *bitset = NULL; - /* Allocate the bitset itself. */ - if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; +typedef struct art_internal_validate_s { + const char **reason; + art_validate_cb_t validate_cb; + + int depth; + art_key_chunk_t current_key[ART_KEY_BYTES]; +} art_internal_validate_t; + +// Set the reason message, and return false for convenience. +static inline bool art_validate_fail(const art_internal_validate_t *validate, + const char *msg) { + *validate->reason = msg; + return false; +} + +// Inner node, with prefix. +// +// We use a fixed-length array as a pointer would be larger than the array. +typedef struct art_inner_node_s { + art_typecode_t typecode; + uint8_t prefix_size; + uint8_t prefix[ART_KEY_BYTES - 1]; +} art_inner_node_t; + +// Inner node types. + +// Node4: key[i] corresponds with children[i]. Keys are sorted. +typedef struct art_node4_s { + art_inner_node_t base; + uint8_t count; + uint8_t keys[4]; + art_node_t *children[4]; +} art_node4_t; + +// Node16: key[i] corresponds with children[i]. Keys are sorted. +typedef struct art_node16_s { + art_inner_node_t base; + uint8_t count; + uint8_t keys[16]; + art_node_t *children[16]; +} art_node16_t; + +// Node48: key[i] corresponds with children[key[i]] if key[i] != +// CROARING_ART_NODE48_EMPTY_VAL. Keys are naturally sorted due to direct +// indexing. +typedef struct art_node48_s { + art_inner_node_t base; + uint8_t count; + // Bitset where the ith bit is set if children[i] is available + // Because there are at most 48 children, only the bottom 48 bits are used. + uint64_t available_children; + uint8_t keys[256]; + art_node_t *children[48]; +} art_node48_t; + +// Node256: children[i] is directly indexed by key chunk. A child is present if +// children[i] != NULL. +typedef struct art_node256_s { + art_inner_node_t base; + uint16_t count; + art_node_t *children[256]; +} art_node256_t; + +// Helper struct to refer to a child within a node at a specific index. +typedef struct art_indexed_child_s { + art_node_t *child; + uint8_t index; + art_key_chunk_t key_chunk; +} art_indexed_child_t; + +static inline bool art_is_leaf(const art_node_t *node) { + return CROARING_IS_LEAF(node); +} + +static void art_leaf_populate(art_leaf_t *leaf, const art_key_chunk_t key[]) { + memcpy(leaf->key, key, ART_KEY_BYTES); +} + +static inline uint8_t art_get_type(const art_inner_node_t *node) { + return node->typecode; +} + +static inline void art_init_inner_node(art_inner_node_t *node, + art_typecode_t typecode, + const art_key_chunk_t prefix[], + uint8_t prefix_size) { + node->typecode = typecode; + node->prefix_size = prefix_size; + memcpy(node->prefix, prefix, prefix_size * sizeof(art_key_chunk_t)); +} + +static void art_free_node(art_node_t *node); + +// ===================== Start of node-specific functions ====================== + +static art_node4_t *art_node4_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node16_t *art_node16_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node48_t *art_node48_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node256_t *art_node256_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); + +static art_node_t *art_node4_insert(art_node4_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node16_insert(art_node16_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node48_insert(art_node48_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node256_insert(art_node256_t *node, art_node_t *child, + uint8_t key); + +static art_node4_t *art_node4_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node4_t *node = (art_node4_t *)roaring_malloc(sizeof(art_node4_t)); + art_init_inner_node(&node->base, CROARING_ART_NODE4_TYPE, prefix, + prefix_size); + node->count = 0; + return node; +} + +static void art_free_node4(art_node4_t *node) { + for (size_t i = 0; i < node->count; ++i) { + art_free_node(node->children[i]); + } + roaring_free(node); +} + +static inline art_node_t *art_node4_find_child(const art_node4_t *node, + art_key_chunk_t key) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key) { + return node->children[i]; + } } - bitset->array = NULL; - bitset->arraysize = 0; - bitset->capacity = 0; - return bitset; + return NULL; } -/* Create a new bitset able to contain size bits. Return NULL in case of - * failure. */ -bitset_t *bitset_create_with_capacity(size_t size) { - bitset_t *bitset = NULL; - /* Allocate the bitset itself. */ - if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; +static art_node_t *art_node4_insert(art_node4_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 4) { + size_t idx = 0; + for (; idx < node->count; ++idx) { + if (node->keys[idx] > key) { + break; + } + } + size_t after = node->count - idx; + // Shift other keys to maintain sorted order. + memmove(node->keys + idx + 1, node->keys + idx, + after * sizeof(art_key_chunk_t)); + memmove(node->children + idx + 1, node->children + idx, + after * sizeof(art_node_t *)); + + node->children[idx] = child; + node->keys[idx] = key; + node->count++; + return (art_node_t *)node; + } + art_node16_t *new_node = + art_node16_create(node->base.prefix, node->base.prefix_size); + // Instead of calling insert, this could be specialized to 2x memcpy and + // setting the count. + for (size_t i = 0; i < 4; ++i) { + art_node16_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return art_node16_insert(new_node, child, key); +} + +static inline art_node_t *art_node4_erase(art_node4_t *node, + art_key_chunk_t key_chunk) { + int idx = -1; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + idx = i; + } + } + if (idx == -1) { + return (art_node_t *)node; + } + if (node->count == 2) { + // Only one child remains after erasing, so compress the path by + // removing this node. + uint8_t other_idx = idx ^ 1; + art_node_t *remaining_child = node->children[other_idx]; + art_key_chunk_t remaining_child_key = node->keys[other_idx]; + if (!art_is_leaf(remaining_child)) { + // Correct the prefix of the child node. + art_inner_node_t *inner_node = (art_inner_node_t *)remaining_child; + memmove(inner_node->prefix + node->base.prefix_size + 1, + inner_node->prefix, inner_node->prefix_size); + memcpy(inner_node->prefix, node->base.prefix, + node->base.prefix_size); + inner_node->prefix[node->base.prefix_size] = remaining_child_key; + inner_node->prefix_size += node->base.prefix_size + 1; + } + roaring_free(node); + return remaining_child; + } + // Shift other keys to maintain sorted order. + size_t after_next = node->count - idx - 1; + memmove(node->keys + idx, node->keys + idx + 1, + after_next * sizeof(art_key_chunk_t)); + memmove(node->children + idx, node->children + idx + 1, + after_next * sizeof(art_node_t *)); + node->count--; + return (art_node_t *)node; +} + +static inline void art_node4_replace(art_node4_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + node->children[i] = new_child; + return; + } } - bitset->arraysize = - (size + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8); - bitset->capacity = bitset->arraysize; - if ((bitset->array = - (uint64_t *)roaring_calloc(bitset->arraysize, sizeof(uint64_t))) == NULL) { - roaring_free(bitset); - return NULL; +} + +static inline art_indexed_child_t art_node4_next_child(const art_node4_t *node, + int index) { + art_indexed_child_t indexed_child; + index++; + if (index >= node->count) { + indexed_child.child = NULL; + return indexed_child; } - return bitset; + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; } -/* Create a copy */ -bitset_t *bitset_copy(const bitset_t *bitset) { - bitset_t *copy = NULL; - /* Allocate the bitset itself. */ - if ((copy = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; +static inline art_indexed_child_t art_node4_prev_child(const art_node4_t *node, + int index) { + if (index > node->count) { + index = node->count; } - memcpy(copy, bitset, sizeof(bitset_t)); - copy->capacity = copy->arraysize; - if ((copy->array = (uint64_t *)roaring_malloc(sizeof(uint64_t) * - bitset->arraysize)) == NULL) { - roaring_free(copy); - return NULL; + index--; + art_indexed_child_t indexed_child; + if (index < 0) { + indexed_child.child = NULL; + return indexed_child; } - memcpy(copy->array, bitset->array, sizeof(uint64_t) * bitset->arraysize); - return copy; + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; } -void bitset_clear(bitset_t *bitset) { - memset(bitset->array, 0, sizeof(uint64_t) * bitset->arraysize); +static inline art_indexed_child_t art_node4_child_at(const art_node4_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= node->count) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; } -void bitset_fill(bitset_t *bitset) { - memset(bitset->array, 0xff, sizeof(uint64_t) * bitset->arraysize); +static inline art_indexed_child_t art_node4_lower_bound( + art_node4_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] >= key_chunk) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = node->keys[i]; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; } -void bitset_shift_left(bitset_t *bitset, size_t s) { - size_t extra_words = s / 64; - int inword_shift = s % 64; - size_t as = bitset->arraysize; - if (inword_shift == 0) { - bitset_resize(bitset, as + extra_words, false); - // could be done with a memmove - for (size_t i = as + extra_words; i > extra_words; i--) { - bitset->array[i - 1] = bitset->array[i - 1 - extra_words]; +static bool art_internal_validate_at(const art_node_t *node, + art_internal_validate_t validator); + +static bool art_node4_internal_validate(const art_node4_t *node, + art_internal_validate_t validator) { + if (node->count == 0) { + return art_validate_fail(&validator, "Node4 has no children"); + } + if (node->count > 4) { + return art_validate_fail(&validator, "Node4 has too many children"); + } + if (node->count == 1) { + return art_validate_fail( + &validator, "Node4 and child node should have been combined"); + } + validator.depth++; + for (int i = 0; i < node->count; ++i) { + if (i > 0) { + if (node->keys[i - 1] >= node->keys[i]) { + return art_validate_fail( + &validator, "Node4 keys are not strictly increasing"); + } } - } else { - bitset_resize(bitset, as + extra_words + 1, true); - bitset->array[as + extra_words] = - bitset->array[as - 1] >> (64 - inword_shift); - for (size_t i = as + extra_words; i >= extra_words + 2; i--) { - bitset->array[i - 1] = - (bitset->array[i - 1 - extra_words] << inword_shift) | - (bitset->array[i - 2 - extra_words] >> (64 - inword_shift)); + for (int j = i + 1; j < node->count; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node4 has duplicate children"); + } + } + validator.current_key[validator.depth - 1] = node->keys[i]; + if (!art_internal_validate_at(node->children[i], validator)) { + return false; } - bitset->array[extra_words] = bitset->array[0] << inword_shift; - } - for (size_t i = 0; i < extra_words; i++) { - bitset->array[i] = 0; } + return true; } -void bitset_shift_right(bitset_t *bitset, size_t s) { - size_t extra_words = s / 64; - int inword_shift = s % 64; - size_t as = bitset->arraysize; - if (inword_shift == 0) { - // could be done with a memmove - for (size_t i = 0; i < as - extra_words; i++) { - bitset->array[i] = bitset->array[i + extra_words]; - } - bitset_resize(bitset, as - extra_words, false); +static art_node16_t *art_node16_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node16_t *node = (art_node16_t *)roaring_malloc(sizeof(art_node16_t)); + art_init_inner_node(&node->base, CROARING_ART_NODE16_TYPE, prefix, + prefix_size); + node->count = 0; + return node; +} - } else { - for (size_t i = 0; i + extra_words + 1 < as; i++) { - bitset->array[i] = - (bitset->array[i + extra_words] >> inword_shift) | - (bitset->array[i + extra_words + 1] << (64 - inword_shift)); +static void art_free_node16(art_node16_t *node) { + for (size_t i = 0; i < node->count; ++i) { + art_free_node(node->children[i]); + } + roaring_free(node); +} + +static inline art_node_t *art_node16_find_child(const art_node16_t *node, + art_key_chunk_t key) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key) { + return node->children[i]; } - bitset->array[as - extra_words - 1] = - (bitset->array[as - 1] >> inword_shift); - bitset_resize(bitset, as - extra_words, false); } + return NULL; } -/* Free memory. */ -void bitset_free(bitset_t *bitset) { - if(bitset == NULL) { return; } - roaring_free(bitset->array); - roaring_free(bitset); +static art_node_t *art_node16_insert(art_node16_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 16) { + size_t idx = 0; + for (; idx < node->count; ++idx) { + if (node->keys[idx] > key) { + break; + } + } + size_t after = node->count - idx; + // Shift other keys to maintain sorted order. + memmove(node->keys + idx + 1, node->keys + idx, + after * sizeof(art_key_chunk_t)); + memmove(node->children + idx + 1, node->children + idx, + after * sizeof(art_node_t *)); + + node->children[idx] = child; + node->keys[idx] = key; + node->count++; + return (art_node_t *)node; + } + art_node48_t *new_node = + art_node48_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 16; ++i) { + art_node48_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return art_node48_insert(new_node, child, key); +} + +static inline art_node_t *art_node16_erase(art_node16_t *node, + uint8_t key_chunk) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + // Shift other keys to maintain sorted order. + size_t after_next = node->count - i - 1; + memmove(node->keys + i, node->keys + i + 1, + after_next * sizeof(key_chunk)); + memmove(node->children + i, node->children + i + 1, + after_next * sizeof(art_node_t *)); + node->count--; + break; + } + } + if (node->count > 4) { + return (art_node_t *)node; + } + art_node4_t *new_node = + art_node4_create(node->base.prefix, node->base.prefix_size); + // Instead of calling insert, this could be specialized to 2x memcpy and + // setting the count. + for (size_t i = 0; i < 4; ++i) { + art_node4_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return (art_node_t *)new_node; } -/* Resize the bitset so that it can support newarraysize * 64 bits. Return true - * in case of success, false for failure. */ -bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes) { - if(newarraysize > SIZE_MAX/64) { return false; } - size_t smallest = - newarraysize < bitset->arraysize ? newarraysize : bitset->arraysize; - if (bitset->capacity < newarraysize) { - uint64_t *newarray; - size_t newcapacity = bitset->capacity; - if(newcapacity == 0) { newcapacity = 1; } - while(newcapacity < newarraysize) { newcapacity *= 2; } - if ((newarray = (uint64_t *) roaring_realloc(bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { - return false; +static inline void art_node16_replace(art_node16_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + for (uint8_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + node->children[i] = new_child; + return; } - bitset->capacity = newcapacity; - bitset->array = newarray; } - if (padwithzeroes && (newarraysize > smallest)) - memset(bitset->array + smallest, 0, - sizeof(uint64_t) * (newarraysize - smallest)); - bitset->arraysize = newarraysize; - return true; // success! } -size_t bitset_count(const bitset_t *bitset) { - size_t card = 0; - size_t k = 0; - for (; k + 7 < bitset->arraysize; k += 8) { - card += roaring_hamming(bitset->array[k]); - card += roaring_hamming(bitset->array[k + 1]); - card += roaring_hamming(bitset->array[k + 2]); - card += roaring_hamming(bitset->array[k + 3]); - card += roaring_hamming(bitset->array[k + 4]); - card += roaring_hamming(bitset->array[k + 5]); - card += roaring_hamming(bitset->array[k + 6]); - card += roaring_hamming(bitset->array[k + 7]); +static inline art_indexed_child_t art_node16_next_child( + const art_node16_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + if (index >= node->count) { + indexed_child.child = NULL; + return indexed_child; } - for (; k + 3 < bitset->arraysize; k += 4) { - card += roaring_hamming(bitset->array[k]); - card += roaring_hamming(bitset->array[k + 1]); - card += roaring_hamming(bitset->array[k + 2]); - card += roaring_hamming(bitset->array[k + 3]); + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node16_prev_child( + const art_node16_t *node, int index) { + if (index > node->count) { + index = node->count; } - for (; k < bitset->arraysize; k++) { - card += roaring_hamming(bitset->array[k]); + index--; + art_indexed_child_t indexed_child; + if (index < 0) { + indexed_child.child = NULL; + return indexed_child; } - return card; + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; } -bool bitset_inplace_union(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - for (size_t k = 0; k < minlength; ++k) { - b1->array[k] |= b2->array[k]; - } - if (b2->arraysize > b1->arraysize) { - size_t oldsize = b1->arraysize; - if (!bitset_resize(b1, b2->arraysize, false)) return false; - memcpy(b1->array + oldsize, b2->array + oldsize, - (b2->arraysize - oldsize) * sizeof(uint64_t)); +static inline art_indexed_child_t art_node16_child_at(const art_node16_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= node->count) { + indexed_child.child = NULL; + return indexed_child; } - return true; + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; } -size_t bitset_minimum(const bitset_t *bitset) { - for (size_t k = 0; k < bitset->arraysize; k++) { - uint64_t w = bitset->array[k]; - if (w != 0) { - return roaring_trailing_zeroes(w) + k * 64; +static inline art_indexed_child_t art_node16_lower_bound( + art_node16_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] >= key_chunk) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = node->keys[i]; + return indexed_child; } } - return 0; + indexed_child.child = NULL; + return indexed_child; } -bool bitset_grow(bitset_t *bitset, size_t newarraysize) { - if(newarraysize < bitset->arraysize) { return false; } - if(newarraysize > SIZE_MAX/64) { return false; } - if (bitset->capacity < newarraysize) { - uint64_t *newarray; - size_t newcapacity = (UINT64_C(0xFFFFFFFFFFFFFFFF) >> roaring_leading_zeroes(newarraysize)) + 1; - while(newcapacity < newarraysize) { newcapacity *= 2; } - if ((newarray = (uint64_t *) roaring_realloc(bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { +static bool art_node16_internal_validate(const art_node16_t *node, + art_internal_validate_t validator) { + if (node->count <= 4) { + return art_validate_fail(&validator, "Node16 has too few children"); + } + if (node->count > 16) { + return art_validate_fail(&validator, "Node16 has too many children"); + } + validator.depth++; + for (int i = 0; i < node->count; ++i) { + if (i > 0) { + if (node->keys[i - 1] >= node->keys[i]) { + return art_validate_fail( + &validator, "Node16 keys are not strictly increasing"); + } + } + for (int j = i + 1; j < node->count; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node16 has duplicate children"); + } + } + validator.current_key[validator.depth - 1] = node->keys[i]; + if (!art_internal_validate_at(node->children[i], validator)) { return false; } - bitset->capacity = newcapacity; - bitset->array = newarray; } - memset(bitset->array + bitset->arraysize, 0, - sizeof(uint64_t) * (newarraysize - bitset->arraysize)); - bitset->arraysize = newarraysize; - return true; // success! + return true; } -size_t bitset_maximum(const bitset_t *bitset) { - for (size_t k = bitset->arraysize; k > 0; k--) { - uint64_t w = bitset->array[k - 1]; - if (w != 0) { - return 63 - roaring_leading_zeroes(w) + (k - 1) * 64; - } +static art_node48_t *art_node48_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node48_t *node = (art_node48_t *)roaring_malloc(sizeof(art_node48_t)); + art_init_inner_node(&node->base, CROARING_ART_NODE48_TYPE, prefix, + prefix_size); + node->count = 0; + node->available_children = CROARING_NODE48_AVAILABLE_CHILDREN_MASK; + for (size_t i = 0; i < 256; ++i) { + node->keys[i] = CROARING_ART_NODE48_EMPTY_VAL; } - return 0; + return node; } -/* Returns true if bitsets share no common elements, false otherwise. - * - * Performs early-out if common element found. */ -bool bitsets_disjoint(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; +static void art_free_node48(art_node48_t *node) { + uint64_t used_children = + (node->available_children) ^ CROARING_NODE48_AVAILABLE_CHILDREN_MASK; + while (used_children != 0) { + // We checked above that used_children is not zero + uint8_t child_idx = roaring_trailing_zeroes(used_children); + art_free_node(node->children[child_idx]); + used_children &= ~(UINT64_C(1) << child_idx); + } + roaring_free(node); +} - for (size_t k = 0; k < minlength; k++) { - if ((b1->array[k] & b2->array[k]) != 0) return false; +static inline art_node_t *art_node48_find_child(const art_node48_t *node, + art_key_chunk_t key) { + uint8_t val_idx = node->keys[key]; + if (val_idx != CROARING_ART_NODE48_EMPTY_VAL) { + return node->children[val_idx]; } - return true; + return NULL; } -/* Returns true if bitsets contain at least 1 common element, false if they are - * disjoint. - * - * Performs early-out if common element found. */ -bool bitsets_intersect(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; +static art_node_t *art_node48_insert(art_node48_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 48) { + // node->available_children is only zero when the node is full (count == + // 48), we just checked count < 48 + uint8_t val_idx = roaring_trailing_zeroes(node->available_children); + node->keys[key] = val_idx; + node->children[val_idx] = child; + node->count++; + node->available_children &= ~(UINT64_C(1) << val_idx); + return (art_node_t *)node; + } + art_node256_t *new_node = + art_node256_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + uint8_t val_idx = node->keys[i]; + if (val_idx != CROARING_ART_NODE48_EMPTY_VAL) { + art_node256_insert(new_node, node->children[val_idx], i); + } + } + roaring_free(node); + return art_node256_insert(new_node, child, key); +} - for (size_t k = 0; k < minlength; k++) { - if ((b1->array[k] & b2->array[k]) != 0) return true; +static inline art_node_t *art_node48_erase(art_node48_t *node, + uint8_t key_chunk) { + uint8_t val_idx = node->keys[key_chunk]; + if (val_idx == CROARING_ART_NODE48_EMPTY_VAL) { + return (art_node_t *)node; } - return false; + node->keys[key_chunk] = CROARING_ART_NODE48_EMPTY_VAL; + node->available_children |= UINT64_C(1) << val_idx; + node->count--; + if (node->count > 16) { + return (art_node_t *)node; + } + + art_node16_t *new_node = + art_node16_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + val_idx = node->keys[i]; + if (val_idx != CROARING_ART_NODE48_EMPTY_VAL) { + art_node16_insert(new_node, node->children[val_idx], i); + } + } + roaring_free(node); + return (art_node_t *)new_node; } -/* Returns true if b has any bits set in or after b->array[starting_loc]. */ -static bool any_bits_set(const bitset_t *b, size_t starting_loc) { - if (starting_loc >= b->arraysize) { - return false; +static inline void art_node48_replace(art_node48_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + uint8_t val_idx = node->keys[key_chunk]; + assert(val_idx != CROARING_ART_NODE48_EMPTY_VAL); + node->children[val_idx] = new_child; +} + +static inline art_indexed_child_t art_node48_next_child( + const art_node48_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + for (size_t i = index; i < 256; ++i) { + if (node->keys[i] != CROARING_ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; + } } - for (size_t k = starting_loc; k < b->arraysize; k++) { - if (b->array[k] != 0) return true; + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node48_prev_child( + const art_node48_t *node, int index) { + if (index > 256) { + index = 256; } - return false; + index--; + art_indexed_child_t indexed_child; + for (int i = index; i >= 0; --i) { + if (node->keys[i] != CROARING_ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; } -/* Returns true if b1 has all of b2's bits set. - * - * Performs early out if a bit is found in b2 that is not found in b1. */ -bool bitset_contains_all(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2) { - size_t min_size = b1->arraysize; - if(b1->arraysize > b2->arraysize) { - min_size = b2->arraysize; +static inline art_indexed_child_t art_node48_child_at(const art_node48_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= 256) { + indexed_child.child = NULL; + return indexed_child; } - for (size_t k = 0; k < min_size; k++) { - if ((b1->array[k] & b2->array[k]) != b2->array[k]) { - return false; + indexed_child.index = index; + indexed_child.child = node->children[node->keys[index]]; + indexed_child.key_chunk = index; + return indexed_child; +} + +static inline art_indexed_child_t art_node48_lower_bound( + art_node48_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = key_chunk; i < 256; ++i) { + if (node->keys[i] != CROARING_ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; } } - if (b2->arraysize > b1->arraysize) { - /* Need to check if b2 has any bits set beyond b1's array */ - return !any_bits_set(b2, b1->arraysize); + indexed_child.child = NULL; + return indexed_child; +} + +static bool art_node48_internal_validate(const art_node48_t *node, + art_internal_validate_t validator) { + if (node->count <= 16) { + return art_validate_fail(&validator, "Node48 has too few children"); + } + if (node->count > 48) { + return art_validate_fail(&validator, "Node48 has too many children"); + } + uint64_t used_children = 0; + for (int i = 0; i < 256; ++i) { + uint8_t child_idx = node->keys[i]; + if (child_idx != CROARING_ART_NODE48_EMPTY_VAL) { + if (used_children & (UINT64_C(1) << child_idx)) { + return art_validate_fail( + &validator, "Node48 keys point to the same child index"); + } + + art_node_t *child = node->children[child_idx]; + if (child == NULL) { + return art_validate_fail(&validator, "Node48 has a NULL child"); + } + used_children |= UINT64_C(1) << child_idx; + } + } + uint64_t expected_used_children = + (node->available_children) ^ CROARING_NODE48_AVAILABLE_CHILDREN_MASK; + if (used_children != expected_used_children) { + return art_validate_fail( + &validator, + "Node48 available_children does not match actual children"); + } + while (used_children != 0) { + uint8_t child_idx = roaring_trailing_zeroes(used_children); + used_children &= used_children - 1; + + uint64_t other_children = used_children; + while (other_children != 0) { + uint8_t other_child_idx = roaring_trailing_zeroes(other_children); + if (node->children[child_idx] == node->children[other_child_idx]) { + return art_validate_fail(&validator, + "Node48 has duplicate children"); + } + other_children &= other_children - 1; + } + } + + validator.depth++; + for (int i = 0; i < 256; ++i) { + if (node->keys[i] != CROARING_ART_NODE48_EMPTY_VAL) { + validator.current_key[validator.depth - 1] = i; + if (!art_internal_validate_at(node->children[node->keys[i]], + validator)) { + return false; + } + } } return true; } -size_t bitset_union_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t answer = 0; - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k + 3 < minlength; k += 4) { - answer += roaring_hamming(b1->array[k] | b2->array[k]); - answer += roaring_hamming(b1->array[k + 1] | b2->array[k + 1]); - answer += roaring_hamming(b1->array[k + 2] | b2->array[k + 2]); - answer += roaring_hamming(b1->array[k + 3] | b2->array[k + 3]); +static art_node256_t *art_node256_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node256_t *node = + (art_node256_t *)roaring_malloc(sizeof(art_node256_t)); + art_init_inner_node(&node->base, CROARING_ART_NODE256_TYPE, prefix, + prefix_size); + node->count = 0; + for (size_t i = 0; i < 256; ++i) { + node->children[i] = NULL; } - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] | b2->array[k]); + return node; +} + +static void art_free_node256(art_node256_t *node) { + for (size_t i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + art_free_node(node->children[i]); + } } - if (b2->arraysize > b1->arraysize) { - // k is equal to b1->arraysize - for (; k + 3 < b2->arraysize; k += 4) { - answer += roaring_hamming(b2->array[k]); - answer += roaring_hamming(b2->array[k + 1]); - answer += roaring_hamming(b2->array[k + 2]); - answer += roaring_hamming(b2->array[k + 3]); + roaring_free(node); +} + +static inline art_node_t *art_node256_find_child(const art_node256_t *node, + art_key_chunk_t key) { + return node->children[key]; +} + +static art_node_t *art_node256_insert(art_node256_t *node, art_node_t *child, + uint8_t key) { + node->children[key] = child; + node->count++; + return (art_node_t *)node; +} + +static inline art_node_t *art_node256_erase(art_node256_t *node, + uint8_t key_chunk) { + node->children[key_chunk] = NULL; + node->count--; + if (node->count > 48) { + return (art_node_t *)node; + } + + art_node48_t *new_node = + art_node48_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + art_node48_insert(new_node, node->children[i], i); } - for (; k < b2->arraysize; ++k) { - answer += roaring_hamming(b2->array[k]); + } + roaring_free(node); + return (art_node_t *)new_node; +} + +static inline void art_node256_replace(art_node256_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + node->children[key_chunk] = new_child; +} + +static inline art_indexed_child_t art_node256_next_child( + const art_node256_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + for (size_t i = index; i < 256; ++i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; } - } else { - // k is equal to b2->arraysize - for (; k + 3 < b1->arraysize; k += 4) { - answer += roaring_hamming(b1->array[k]); - answer += roaring_hamming(b1->array[k + 1]); - answer += roaring_hamming(b1->array[k + 2]); - answer += roaring_hamming(b1->array[k + 3]); + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_prev_child( + const art_node256_t *node, int index) { + if (index > 256) { + index = 256; + } + index--; + art_indexed_child_t indexed_child; + for (int i = index; i >= 0; --i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; } - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_child_at( + const art_node256_t *node, int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= 256) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = index; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_lower_bound( + art_node256_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = key_chunk; i < 256; ++i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; } } - return answer; + indexed_child.child = NULL; + return indexed_child; } -void bitset_inplace_intersection(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] &= b2->array[k]; +static bool art_node256_internal_validate(const art_node256_t *node, + art_internal_validate_t validator) { + if (node->count <= 48) { + return art_validate_fail(&validator, "Node256 has too few children"); } - for (; k < b1->arraysize; ++k) { - b1->array[k] = 0; // memset could, maybe, be a tiny bit faster + if (node->count > 256) { + return art_validate_fail(&validator, "Node256 has too many children"); + } + validator.depth++; + int actual_count = 0; + for (int i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + actual_count++; + + for (int j = i + 1; j < 256; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node256 has duplicate children"); + } + } + + validator.current_key[validator.depth - 1] = i; + if (!art_internal_validate_at(node->children[i], validator)) { + return false; + } + } } + if (actual_count != node->count) { + return art_validate_fail( + &validator, "Node256 count does not match actual children"); + } + return true; } -size_t bitset_intersection_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t answer = 0; - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - for (size_t k = 0; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] & b2->array[k]); +// Finds the child with the given key chunk in the inner node, returns NULL if +// no such child is found. +static art_node_t *art_find_child(const art_inner_node_t *node, + art_key_chunk_t key_chunk) { + switch (art_get_type(node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_find_child((art_node4_t *)node, key_chunk); + case CROARING_ART_NODE16_TYPE: + return art_node16_find_child((art_node16_t *)node, key_chunk); + case CROARING_ART_NODE48_TYPE: + return art_node48_find_child((art_node48_t *)node, key_chunk); + case CROARING_ART_NODE256_TYPE: + return art_node256_find_child((art_node256_t *)node, key_chunk); + default: + assert(false); + return NULL; } - return answer; } -void bitset_inplace_difference(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] &= ~(b2->array[k]); +// Replaces the child with the given key chunk in the inner node. +static void art_replace(art_inner_node_t *node, art_key_chunk_t key_chunk, + art_node_t *new_child) { + switch (art_get_type(node)) { + case CROARING_ART_NODE4_TYPE: + art_node4_replace((art_node4_t *)node, key_chunk, new_child); + break; + case CROARING_ART_NODE16_TYPE: + art_node16_replace((art_node16_t *)node, key_chunk, new_child); + break; + case CROARING_ART_NODE48_TYPE: + art_node48_replace((art_node48_t *)node, key_chunk, new_child); + break; + case CROARING_ART_NODE256_TYPE: + art_node256_replace((art_node256_t *)node, key_chunk, new_child); + break; + default: + assert(false); } } -size_t bitset_difference_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - size_t answer = 0; - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] & ~(b2->array[k])); +// Erases the child with the given key chunk from the inner node, returns the +// updated node (the same as the initial node if it was not shrunk). +static art_node_t *art_node_erase(art_inner_node_t *node, + art_key_chunk_t key_chunk) { + switch (art_get_type(node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_erase((art_node4_t *)node, key_chunk); + case CROARING_ART_NODE16_TYPE: + return art_node16_erase((art_node16_t *)node, key_chunk); + case CROARING_ART_NODE48_TYPE: + return art_node48_erase((art_node48_t *)node, key_chunk); + case CROARING_ART_NODE256_TYPE: + return art_node256_erase((art_node256_t *)node, key_chunk); + default: + assert(false); + return NULL; } - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); +} + +// Inserts the leaf with the given key chunk in the inner node, returns a +// pointer to the (possibly expanded) node. +static art_node_t *art_node_insert_leaf(art_inner_node_t *node, + art_key_chunk_t key_chunk, + art_leaf_t *leaf) { + art_node_t *child = (art_node_t *)(CROARING_SET_LEAF(leaf)); + switch (art_get_type(node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_insert((art_node4_t *)node, child, key_chunk); + case CROARING_ART_NODE16_TYPE: + return art_node16_insert((art_node16_t *)node, child, key_chunk); + case CROARING_ART_NODE48_TYPE: + return art_node48_insert((art_node48_t *)node, child, key_chunk); + case CROARING_ART_NODE256_TYPE: + return art_node256_insert((art_node256_t *)node, child, key_chunk); + default: + assert(false); + return NULL; } - return answer; } -bool bitset_inplace_symmetric_difference(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] ^= b2->array[k]; +// Frees the node and its children. Leaves are freed by the user. +static void art_free_node(art_node_t *node) { + if (art_is_leaf(node)) { + // We leave it up to the user to free leaves. + return; } - if (b2->arraysize > b1->arraysize) { - size_t oldsize = b1->arraysize; - if (!bitset_resize(b1, b2->arraysize, false)) return false; - memcpy(b1->array + oldsize, b2->array + oldsize, - (b2->arraysize - oldsize) * sizeof(uint64_t)); + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + art_free_node4((art_node4_t *)node); + break; + case CROARING_ART_NODE16_TYPE: + art_free_node16((art_node16_t *)node); + break; + case CROARING_ART_NODE48_TYPE: + art_free_node48((art_node48_t *)node); + break; + case CROARING_ART_NODE256_TYPE: + art_free_node256((art_node256_t *)node); + break; + default: + assert(false); } - return true; } -size_t bitset_symmetric_difference_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - size_t answer = 0; - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] ^ b2->array[k]); +// Returns the next child in key order, or NULL if called on a leaf. +// Provided index may be in the range [-1, 255]. +static art_indexed_child_t art_node_next_child(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_next_child((art_node4_t *)node, index); + case CROARING_ART_NODE16_TYPE: + return art_node16_next_child((art_node16_t *)node, index); + case CROARING_ART_NODE48_TYPE: + return art_node48_next_child((art_node48_t *)node, index); + case CROARING_ART_NODE256_TYPE: + return art_node256_next_child((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the previous child in key order, or NULL if called on a leaf. +// Provided index may be in the range [0, 256]. +static art_indexed_child_t art_node_prev_child(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_prev_child((art_node4_t *)node, index); + case CROARING_ART_NODE16_TYPE: + return art_node16_prev_child((art_node16_t *)node, index); + case CROARING_ART_NODE48_TYPE: + return art_node48_prev_child((art_node48_t *)node, index); + case CROARING_ART_NODE256_TYPE: + return art_node256_prev_child((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the child found at the provided index, or NULL if called on a leaf. +// Provided index is only valid if returned by art_node_(next|prev)_child. +static art_indexed_child_t art_node_child_at(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_child_at((art_node4_t *)node, index); + case CROARING_ART_NODE16_TYPE: + return art_node16_child_at((art_node16_t *)node, index); + case CROARING_ART_NODE48_TYPE: + return art_node48_child_at((art_node48_t *)node, index); + case CROARING_ART_NODE256_TYPE: + return art_node256_child_at((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the child with the smallest key equal to or greater than the given +// key chunk, NULL if called on a leaf or no such child was found. +static art_indexed_child_t art_node_lower_bound(const art_node_t *node, + art_key_chunk_t key_chunk) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + return art_node4_lower_bound((art_node4_t *)node, key_chunk); + case CROARING_ART_NODE16_TYPE: + return art_node16_lower_bound((art_node16_t *)node, key_chunk); + case CROARING_ART_NODE48_TYPE: + return art_node48_lower_bound((art_node48_t *)node, key_chunk); + case CROARING_ART_NODE256_TYPE: + return art_node256_lower_bound((art_node256_t *)node, key_chunk); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// ====================== End of node-specific functions ======================= + +// Compares the given ranges of two keys, returns their relative order: +// * Key range 1 < key range 2: a negative value +// * Key range 1 == key range 2: 0 +// * Key range 1 > key range 2: a positive value +static inline int art_compare_prefix(const art_key_chunk_t key1[], + uint8_t key1_from, + const art_key_chunk_t key2[], + uint8_t key2_from, uint8_t length) { + return memcmp(key1 + key1_from, key2 + key2_from, length); +} + +// Compares two keys in full, see art_compare_prefix. +int art_compare_keys(const art_key_chunk_t key1[], + const art_key_chunk_t key2[]) { + return art_compare_prefix(key1, 0, key2, 0, ART_KEY_BYTES); +} + +// Returns the length of the common prefix between two key ranges. +static uint8_t art_common_prefix(const art_key_chunk_t key1[], + uint8_t key1_from, uint8_t key1_to, + const art_key_chunk_t key2[], + uint8_t key2_from, uint8_t key2_to) { + uint8_t min_len = key1_to - key1_from; + uint8_t key2_len = key2_to - key2_from; + if (key2_len < min_len) { + min_len = key2_len; + } + uint8_t offset = 0; + for (; offset < min_len; ++offset) { + if (key1[key1_from + offset] != key2[key2_from + offset]) { + return offset; + } + } + return offset; +} + +// Returns a pointer to the rootmost node where the value was inserted, may not +// be equal to `node`. +static art_node_t *art_insert_at(art_node_t *node, const art_key_chunk_t key[], + uint8_t depth, art_leaf_t *new_leaf) { + if (art_is_leaf(node)) { + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + uint8_t common_prefix = art_common_prefix( + leaf->key, depth, ART_KEY_BYTES, key, depth, ART_KEY_BYTES); + + // Previously this was a leaf, create an inner node instead and add both + // the existing and new leaf to it. + art_node_t *new_node = + (art_node_t *)art_node4_create(key + depth, common_prefix); + + new_node = art_node_insert_leaf((art_inner_node_t *)new_node, + leaf->key[depth + common_prefix], leaf); + new_node = art_node_insert_leaf((art_inner_node_t *)new_node, + key[depth + common_prefix], new_leaf); + + // The new inner node is now the rootmost node. + return new_node; + } + art_inner_node_t *inner_node = (art_inner_node_t *)node; + // Not a leaf: inner node + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, key, + depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + // Partial prefix match. Create a new internal node to hold the common + // prefix. + art_node4_t *node4 = + art_node4_create(inner_node->prefix, common_prefix); + + // Make the existing internal node a child of the new internal node. + node4 = (art_node4_t *)art_node4_insert( + node4, node, inner_node->prefix[common_prefix]); + + // Correct the prefix of the moved internal node, trimming off the chunk + // inserted into the new internal node. + inner_node->prefix_size = inner_node->prefix_size - common_prefix - 1; + if (inner_node->prefix_size > 0) { + // Move the remaining prefix to the correct position. + memmove(inner_node->prefix, inner_node->prefix + common_prefix + 1, + inner_node->prefix_size); + } + + // Insert the value in the new internal node. + return art_node_insert_leaf(&node4->base, key[common_prefix + depth], + new_leaf); + } + // Prefix matches entirely or node has no prefix. Look for an existing + // child. + art_key_chunk_t key_chunk = key[depth + common_prefix]; + art_node_t *child = art_find_child(inner_node, key_chunk); + if (child != NULL) { + art_node_t *new_child = + art_insert_at(child, key, depth + common_prefix + 1, new_leaf); + if (new_child != child) { + // Node type changed. + art_replace(inner_node, key_chunk, new_child); + } + return node; + } + return art_node_insert_leaf(inner_node, key_chunk, new_leaf); +} + +// Erase helper struct. +typedef struct art_erase_result_s { + // The rootmost node where the value was erased, may not be equal to `node`. + // If no value was removed, this is null. + art_node_t *rootmost_node; + + // Value removed, null if not removed. + art_val_t *value_erased; +} art_erase_result_t; + +// Searches for the given key starting at `node`, erases it if found. +static art_erase_result_t art_erase_at(art_node_t *node, + const art_key_chunk_t *key, + uint8_t depth) { + art_erase_result_t result; + result.rootmost_node = NULL; + result.value_erased = NULL; + + if (art_is_leaf(node)) { + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + uint8_t common_prefix = art_common_prefix(leaf->key, 0, ART_KEY_BYTES, + key, 0, ART_KEY_BYTES); + if (common_prefix != ART_KEY_BYTES) { + // Leaf key mismatch. + return result; + } + result.value_erased = (art_val_t *)leaf; + return result; + } + art_inner_node_t *inner_node = (art_inner_node_t *)node; + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, key, + depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + // Prefix mismatch. + return result; + } + art_key_chunk_t key_chunk = key[depth + common_prefix]; + art_node_t *child = art_find_child(inner_node, key_chunk); + if (child == NULL) { + // No child with key chunk. + return result; + } + // Try to erase the key further down. Skip the key chunk associated with the + // child in the node. + art_erase_result_t child_result = + art_erase_at(child, key, depth + common_prefix + 1); + if (child_result.value_erased == NULL) { + return result; + } + result.value_erased = child_result.value_erased; + result.rootmost_node = node; + if (child_result.rootmost_node == NULL) { + // Child node was fully erased, erase it from this node's children. + result.rootmost_node = art_node_erase(inner_node, key_chunk); + } else if (child_result.rootmost_node != child) { + // Child node was not fully erased, update the pointer to it in this + // node. + art_replace(inner_node, key_chunk, child_result.rootmost_node); } - if (b2->arraysize > b1->arraysize) { - for (; k < b2->arraysize; ++k) { - answer += roaring_hamming(b2->array[k]); + return result; +} + +// Searches for the given key starting at `node`, returns NULL if the key was +// not found. +static art_val_t *art_find_at(const art_node_t *node, + const art_key_chunk_t *key, uint8_t depth) { + while (!art_is_leaf(node)) { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, + key, depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + return NULL; } - } else { - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); + art_node_t *child = + art_find_child(inner_node, key[depth + inner_node->prefix_size]); + if (child == NULL) { + return NULL; } + node = child; + // Include both the prefix and the child key chunk in the depth. + depth += inner_node->prefix_size + 1; } - return answer; + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + if (depth >= ART_KEY_BYTES) { + return (art_val_t *)leaf; + } + uint8_t common_prefix = + art_common_prefix(leaf->key, 0, ART_KEY_BYTES, key, 0, ART_KEY_BYTES); + if (common_prefix == ART_KEY_BYTES) { + return (art_val_t *)leaf; + } + return NULL; } -bool bitset_trim(bitset_t *bitset) { - size_t newsize = bitset->arraysize; - while (newsize > 0) { - if (bitset->array[newsize - 1] == 0) - newsize -= 1; - else +// Returns the size in bytes of the subtrie. +size_t art_size_in_bytes_at(const art_node_t *node) { + if (art_is_leaf(node)) { + return 0; + } + size_t size = 0; + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: { + size += sizeof(art_node4_t); + } break; + case CROARING_ART_NODE16_TYPE: { + size += sizeof(art_node16_t); + } break; + case CROARING_ART_NODE48_TYPE: { + size += sizeof(art_node48_t); + } break; + case CROARING_ART_NODE256_TYPE: { + size += sizeof(art_node256_t); + } break; + default: + assert(false); break; } - if (bitset->capacity == newsize) return true; // nothing to do - uint64_t *newarray; - if ((newarray = (uint64_t *)roaring_realloc( - bitset->array, sizeof(uint64_t) * newsize)) == NULL) { - return false; + art_indexed_child_t indexed_child = art_node_next_child(node, -1); + while (indexed_child.child != NULL) { + size += art_size_in_bytes_at(indexed_child.child); + indexed_child = art_node_next_child(node, indexed_child.index); } - bitset->array = newarray; - bitset->capacity = newsize; - bitset->arraysize = newsize; - return true; + return size; } +static void art_node_print_type(const art_node_t *node) { + if (art_is_leaf(node)) { + printf("Leaf"); + return; + } + switch (art_get_type((art_inner_node_t *)node)) { + case CROARING_ART_NODE4_TYPE: + printf("Node4"); + return; + case CROARING_ART_NODE16_TYPE: + printf("Node16"); + return; + case CROARING_ART_NODE48_TYPE: + printf("Node48"); + return; + case CROARING_ART_NODE256_TYPE: + printf("Node256"); + return; + default: + assert(false); + return; + } +} -#ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +void art_node_printf(const art_node_t *node, uint8_t depth) { + if (art_is_leaf(node)) { + printf("{ type: Leaf, key: "); + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + for (size_t i = 0; i < ART_KEY_BYTES; ++i) { + printf("%02x", leaf->key[i]); + } + printf(" }\n"); + return; + } + printf("{\n"); + depth++; + + printf("%*s", depth, ""); + printf("type: "); + art_node_print_type(node); + printf("\n"); + + art_inner_node_t *inner_node = (art_inner_node_t *)node; + printf("%*s", depth, ""); + printf("prefix_size: %d\n", inner_node->prefix_size); + + printf("%*s", depth, ""); + printf("prefix: "); + for (uint8_t i = 0; i < inner_node->prefix_size; ++i) { + printf("%02x", inner_node->prefix[i]); + } + printf("\n"); + + switch (art_get_type(inner_node)) { + case CROARING_ART_NODE4_TYPE: { + art_node4_t *node4 = (art_node4_t *)node; + for (uint8_t i = 0; i < node4->count; ++i) { + printf("%*s", depth, ""); + printf("key: %02x ", node4->keys[i]); + art_node_printf(node4->children[i], depth); + } + } break; + case CROARING_ART_NODE16_TYPE: { + art_node16_t *node16 = (art_node16_t *)node; + for (uint8_t i = 0; i < node16->count; ++i) { + printf("%*s", depth, ""); + printf("key: %02x ", node16->keys[i]); + art_node_printf(node16->children[i], depth); + } + } break; + case CROARING_ART_NODE48_TYPE: { + art_node48_t *node48 = (art_node48_t *)node; + for (int i = 0; i < 256; ++i) { + if (node48->keys[i] != CROARING_ART_NODE48_EMPTY_VAL) { + printf("%*s", depth, ""); + printf("key: %02x ", i); + printf("child: %02x ", node48->keys[i]); + art_node_printf(node48->children[node48->keys[i]], depth); + } + } + } break; + case CROARING_ART_NODE256_TYPE: { + art_node256_t *node256 = (art_node256_t *)node; + for (int i = 0; i < 256; ++i) { + if (node256->children[i] != NULL) { + printf("%*s", depth, ""); + printf("key: %02x ", i); + art_node_printf(node256->children[i], depth); + } + } + } break; + default: + assert(false); + break; + } + depth--; + printf("%*s", depth, ""); + printf("}\n"); +} + +void art_insert(art_t *art, const art_key_chunk_t *key, art_val_t *val) { + art_leaf_t *leaf = (art_leaf_t *)val; + art_leaf_populate(leaf, key); + if (art->root == NULL) { + art->root = (art_node_t *)CROARING_SET_LEAF(leaf); + return; + } + art->root = art_insert_at(art->root, key, 0, leaf); +} + +art_val_t *art_erase(art_t *art, const art_key_chunk_t *key) { + if (art->root == NULL) { + return NULL; + } + art_erase_result_t result = art_erase_at(art->root, key, 0); + if (result.value_erased == NULL) { + return NULL; + } + art->root = result.rootmost_node; + return result.value_erased; +} + +art_val_t *art_find(const art_t *art, const art_key_chunk_t *key) { + if (art->root == NULL) { + return NULL; + } + return art_find_at(art->root, key, 0); +} + +bool art_is_empty(const art_t *art) { return art->root == NULL; } + +void art_free(art_t *art) { + if (art->root == NULL) { + return; + } + art_free_node(art->root); +} + +size_t art_size_in_bytes(const art_t *art) { + size_t size = sizeof(art_t); + if (art->root != NULL) { + size += art_size_in_bytes_at(art->root); + } + return size; +} + +void art_printf(const art_t *art) { + if (art->root == NULL) { + return; + } + art_node_printf(art->root, 0); +} + +// Returns the current node that the iterator is positioned at. +static inline art_node_t *art_iterator_node(art_iterator_t *iterator) { + return iterator->frames[iterator->frame].node; +} + +// Sets the iterator key and value to the leaf's key and value. Always returns +// true for convenience. +static inline bool art_iterator_valid_loc(art_iterator_t *iterator, + art_leaf_t *leaf) { + iterator->frames[iterator->frame].node = CROARING_SET_LEAF(leaf); + iterator->frames[iterator->frame].index_in_node = 0; + memcpy(iterator->key, leaf->key, ART_KEY_BYTES); + iterator->value = (art_val_t *)leaf; + return true; +} + +// Invalidates the iterator key and value. Always returns false for convenience. +static inline bool art_iterator_invalid_loc(art_iterator_t *iterator) { + memset(iterator->key, 0, ART_KEY_BYTES); + iterator->value = NULL; + return false; +} + +// Moves the iterator one level down in the tree, given a node at the current +// level and the index of the child that we're going down to. +// +// Note: does not set the index at the new level. +static void art_iterator_down(art_iterator_t *iterator, + const art_inner_node_t *node, + uint8_t index_in_node) { + iterator->frames[iterator->frame].node = (art_node_t *)node; + iterator->frames[iterator->frame].index_in_node = index_in_node; + iterator->frame++; + art_indexed_child_t indexed_child = + art_node_child_at((art_node_t *)node, index_in_node); + assert(indexed_child.child != NULL); + iterator->frames[iterator->frame].node = indexed_child.child; + iterator->depth += node->prefix_size + 1; +} + +// Moves the iterator to the next/previous child of the current node. Returns +// the child moved to, or NULL if there is no neighboring child. +static art_node_t *art_iterator_neighbor_child( + art_iterator_t *iterator, const art_inner_node_t *inner_node, + bool forward) { + art_iterator_frame_t frame = iterator->frames[iterator->frame]; + art_indexed_child_t indexed_child; + if (forward) { + indexed_child = art_node_next_child(frame.node, frame.index_in_node); + } else { + indexed_child = art_node_prev_child(frame.node, frame.index_in_node); + } + if (indexed_child.child != NULL) { + art_iterator_down(iterator, inner_node, indexed_child.index); + } + return indexed_child.child; +} + +// Moves the iterator one level up in the tree, returns false if not possible. +static bool art_iterator_up(art_iterator_t *iterator) { + if (iterator->frame == 0) { + return false; + } + iterator->frame--; + // We went up, so we are at an inner node. + iterator->depth -= + ((art_inner_node_t *)art_iterator_node(iterator))->prefix_size + 1; + return true; +} + +// Moves the iterator one level, followed by a move to the next / previous leaf. +// Sets the status of the iterator. +static bool art_iterator_up_and_move(art_iterator_t *iterator, bool forward) { + if (!art_iterator_up(iterator)) { + // We're at the root. + return art_iterator_invalid_loc(iterator); + } + return art_iterator_move(iterator, forward); +} + +// Initializes the iterator at the first / last leaf of the given node. +// Returns true for convenience. +static bool art_node_init_iterator(const art_node_t *node, + art_iterator_t *iterator, bool first) { + while (!art_is_leaf(node)) { + art_indexed_child_t indexed_child; + if (first) { + indexed_child = art_node_next_child(node, -1); + } else { + indexed_child = art_node_prev_child(node, 256); + } + art_iterator_down(iterator, (art_inner_node_t *)node, + indexed_child.index); + node = indexed_child.child; + } + // We're at a leaf. + iterator->frames[iterator->frame].node = (art_node_t *)node; + iterator->frames[iterator->frame].index_in_node = 0; // Should not matter. + return art_iterator_valid_loc(iterator, CROARING_CAST_LEAF(node)); +} + +bool art_iterator_move(art_iterator_t *iterator, bool forward) { + if (art_is_leaf(art_iterator_node(iterator))) { + bool went_up = art_iterator_up(iterator); + if (!went_up) { + // This leaf is the root, we're done. + return art_iterator_invalid_loc(iterator); + } + } + // Advance within inner node. + art_node_t *neighbor_child = art_iterator_neighbor_child( + iterator, (art_inner_node_t *)art_iterator_node(iterator), forward); + if (neighbor_child != NULL) { + // There is another child at this level, go down to the first or last + // leaf. + return art_node_init_iterator(neighbor_child, iterator, forward); + } + // No more children at this level, go up. + return art_iterator_up_and_move(iterator, forward); +} + +// Assumes the iterator is positioned at a node with an equal prefix path up to +// the depth of the iterator. +static bool art_node_iterator_lower_bound(const art_node_t *node, + art_iterator_t *iterator, + const art_key_chunk_t key[]) { + while (!art_is_leaf(node)) { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + int prefix_comparison = + art_compare_prefix(inner_node->prefix, 0, key, iterator->depth, + inner_node->prefix_size); + if (prefix_comparison < 0) { + // Prefix so far has been equal, but we've found a smaller key. + // Since we take the lower bound within each node, we can return the + // next leaf. + return art_iterator_up_and_move(iterator, true); + } else if (prefix_comparison > 0) { + // No key equal to the key we're looking for, return the first leaf. + return art_node_init_iterator(node, iterator, true); + } + // Prefix is equal, move to lower bound child. + art_key_chunk_t key_chunk = + key[iterator->depth + inner_node->prefix_size]; + art_indexed_child_t indexed_child = + art_node_lower_bound(node, key_chunk); + if (indexed_child.child == NULL) { + // Only smaller keys among children. + return art_iterator_up_and_move(iterator, true); + } + if (indexed_child.key_chunk > key_chunk) { + // Only larger children, return the first larger child. + art_iterator_down(iterator, inner_node, indexed_child.index); + return art_node_init_iterator(indexed_child.child, iterator, true); + } + // We found a child with an equal prefix. + art_iterator_down(iterator, inner_node, indexed_child.index); + node = indexed_child.child; + } + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + if (art_compare_keys(leaf->key, key) >= 0) { + // Leaf has an equal or larger key. + return art_iterator_valid_loc(iterator, leaf); + } + // Leaf has an equal prefix, but the full key is smaller. Move to the next + // leaf. + return art_iterator_up_and_move(iterator, true); +} + +art_iterator_t art_init_iterator(const art_t *art, bool first) { + art_iterator_t iterator = {0}; + if (art->root == NULL) { + return iterator; + } + art_node_init_iterator(art->root, &iterator, first); + return iterator; +} + +bool art_iterator_next(art_iterator_t *iterator) { + return art_iterator_move(iterator, true); +} + +bool art_iterator_prev(art_iterator_t *iterator) { + return art_iterator_move(iterator, false); +} + +bool art_iterator_lower_bound(art_iterator_t *iterator, + const art_key_chunk_t *key) { + if (iterator->value == NULL) { + // We're beyond the end / start of the ART so the iterator does not have + // a valid key. Start from the root. + iterator->frame = 0; + iterator->depth = 0; + return art_node_iterator_lower_bound(art_iterator_node(iterator), + iterator, key); + } + int compare_result = + art_compare_prefix(iterator->key, 0, key, 0, ART_KEY_BYTES); + // Move up until we have an equal prefix, after which we can do a normal + // lower bound search. + while (compare_result != 0) { + if (!art_iterator_up(iterator)) { + if (compare_result < 0) { + // Only smaller keys found. + return art_iterator_invalid_loc(iterator); + } else { + return art_node_init_iterator(art_iterator_node(iterator), + iterator, true); + } + } + // Since we're only moving up, we can keep comparing against the + // iterator key. + art_inner_node_t *inner_node = + (art_inner_node_t *)art_iterator_node(iterator); + compare_result = + art_compare_prefix(iterator->key, 0, key, 0, + iterator->depth + inner_node->prefix_size); + } + if (compare_result > 0) { + return art_node_init_iterator(art_iterator_node(iterator), iterator, + true); + } + return art_node_iterator_lower_bound(art_iterator_node(iterator), iterator, + key); +} + +art_iterator_t art_lower_bound(const art_t *art, const art_key_chunk_t *key) { + art_iterator_t iterator = {0}; + if (art->root != NULL) { + art_node_iterator_lower_bound(art->root, &iterator, key); + } + return iterator; +} + +art_iterator_t art_upper_bound(const art_t *art, const art_key_chunk_t *key) { + art_iterator_t iterator = {0}; + if (art->root != NULL) { + if (art_node_iterator_lower_bound(art->root, &iterator, key) && + art_compare_keys(iterator.key, key) == 0) { + art_iterator_next(&iterator); + } + } + return iterator; +} + +void art_iterator_insert(art_t *art, art_iterator_t *iterator, + const art_key_chunk_t *key, art_val_t *val) { + // TODO: This can likely be faster. + art_insert(art, key, val); + assert(art->root != NULL); + iterator->frame = 0; + iterator->depth = 0; + art_node_iterator_lower_bound(art->root, iterator, key); +} + +// TODO: consider keeping `art_t *art` in the iterator. +art_val_t *art_iterator_erase(art_t *art, art_iterator_t *iterator) { + if (iterator->value == NULL) { + return NULL; + } + art_key_chunk_t initial_key[ART_KEY_BYTES]; + memcpy(initial_key, iterator->key, ART_KEY_BYTES); + + art_val_t *value_erased = iterator->value; + bool went_up = art_iterator_up(iterator); + if (!went_up) { + // We're erasing the root. + art->root = NULL; + art_iterator_invalid_loc(iterator); + return value_erased; + } + + // Erase the leaf. + art_inner_node_t *parent_node = + (art_inner_node_t *)art_iterator_node(iterator); + art_key_chunk_t key_chunk_in_parent = + iterator->key[iterator->depth + parent_node->prefix_size]; + art_node_t *new_parent_node = + art_node_erase(parent_node, key_chunk_in_parent); + + if (new_parent_node != ((art_node_t *)parent_node)) { + // Replace the pointer to the inner node we erased from in its + // parent (it may be a leaf now). + iterator->frames[iterator->frame].node = new_parent_node; + went_up = art_iterator_up(iterator); + if (went_up) { + art_inner_node_t *grandparent_node = + (art_inner_node_t *)art_iterator_node(iterator); + art_key_chunk_t key_chunk_in_grandparent = + iterator->key[iterator->depth + grandparent_node->prefix_size]; + art_replace(grandparent_node, key_chunk_in_grandparent, + new_parent_node); + } else { + // We were already at the rootmost node. + art->root = new_parent_node; + } + } + + iterator->frame = 0; + iterator->depth = 0; + // Do a lower bound search for the initial key, which will find the first + // greater key if it exists. This can likely be mildly faster if we instead + // start from the current position. + art_node_iterator_lower_bound(art->root, iterator, initial_key); + return value_erased; +} + +static bool art_internal_validate_at(const art_node_t *node, + art_internal_validate_t validator) { + if (node == NULL) { + return art_validate_fail(&validator, "node is null"); + } + if (art_is_leaf(node)) { + art_leaf_t *leaf = CROARING_CAST_LEAF(node); + if (art_compare_prefix(leaf->key, 0, validator.current_key, 0, + validator.depth) != 0) { + return art_validate_fail( + &validator, + "leaf key does not match its position's prefix in the tree"); + } + if (validator.validate_cb != NULL && + !validator.validate_cb(leaf, validator.reason)) { + if (*validator.reason == NULL) { + *validator.reason = "leaf validation failed"; + } + return false; + } + } else { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + + if (validator.depth + inner_node->prefix_size + 1 > ART_KEY_BYTES) { + return art_validate_fail(&validator, + "node has too much prefix at given depth"); + } + memcpy(validator.current_key + validator.depth, inner_node->prefix, + inner_node->prefix_size); + validator.depth += inner_node->prefix_size; + + switch (inner_node->typecode) { + case CROARING_ART_NODE4_TYPE: + if (!art_node4_internal_validate((art_node4_t *)inner_node, + validator)) { + return false; + } + break; + case CROARING_ART_NODE16_TYPE: + if (!art_node16_internal_validate((art_node16_t *)inner_node, + validator)) { + return false; + } + break; + case CROARING_ART_NODE48_TYPE: + if (!art_node48_internal_validate((art_node48_t *)inner_node, + validator)) { + return false; + } + break; + case CROARING_ART_NODE256_TYPE: + if (!art_node256_internal_validate((art_node256_t *)inner_node, + validator)) { + return false; + } + break; + default: + return art_validate_fail(&validator, "invalid node type"); + } + } + return true; +} + +bool art_internal_validate(const art_t *art, const char **reason, + art_validate_cb_t validate_cb) { + const char *reason_local; + if (reason == NULL) { + // Always allow assigning through *reason + reason = &reason_local; + } + *reason = NULL; + if (art->root == NULL) { + return true; + } + art_internal_validate_t validator = { + .reason = reason, + .validate_cb = validate_cb, + .depth = 0, + .current_key = {0}, + }; + return art_internal_validate_at(art->root, validator); +} + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace internal +#endif +/* end file src/art/art.c */ +/* begin file src/bitset.c */ +#include +#include +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline void bitset_print(const bitset_t *b); +extern inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, + void *ptr); +extern inline size_t bitset_next_set_bits(const bitset_t *bitset, + size_t *buffer, size_t capacity, + size_t *startfrom); +extern inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag); +extern inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i); +extern inline void bitset_set(bitset_t *bitset, size_t i); +extern inline bool bitset_get(const bitset_t *bitset, size_t i); +extern inline size_t bitset_size_in_words(const bitset_t *bitset); +extern inline size_t bitset_size_in_bits(const bitset_t *bitset); +extern inline size_t bitset_size_in_bytes(const bitset_t *bitset); + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_t *bitset_create(void) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + bitset->array = NULL; + bitset->arraysize = 0; + bitset->capacity = 0; + return bitset; +} + +/* Create a new bitset able to contain size bits. Return NULL in case of + * failure. */ +bitset_t *bitset_create_with_capacity(size_t size) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + bitset->arraysize = + (size + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8); + bitset->capacity = bitset->arraysize; + if ((bitset->array = (uint64_t *)roaring_calloc( + bitset->arraysize, sizeof(uint64_t))) == NULL) { + roaring_free(bitset); + return NULL; + } + return bitset; +} + +/* Create a copy */ +bitset_t *bitset_copy(const bitset_t *bitset) { + bitset_t *copy = NULL; + /* Allocate the bitset itself. */ + if ((copy = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + memcpy(copy, bitset, sizeof(bitset_t)); + copy->capacity = copy->arraysize; + if ((copy->array = (uint64_t *)roaring_malloc(sizeof(uint64_t) * + bitset->arraysize)) == NULL) { + roaring_free(copy); + return NULL; + } + memcpy(copy->array, bitset->array, sizeof(uint64_t) * bitset->arraysize); + return copy; +} + +void bitset_clear(bitset_t *bitset) { + memset(bitset->array, 0, sizeof(uint64_t) * bitset->arraysize); +} + +void bitset_fill(bitset_t *bitset) { + memset(bitset->array, 0xff, sizeof(uint64_t) * bitset->arraysize); +} + +void bitset_shift_left(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + bitset_resize(bitset, as + extra_words, false); + // could be done with a memmove + for (size_t i = as + extra_words; i > extra_words; i--) { + bitset->array[i - 1] = bitset->array[i - 1 - extra_words]; + } + } else { + bitset_resize(bitset, as + extra_words + 1, true); + bitset->array[as + extra_words] = + bitset->array[as - 1] >> (64 - inword_shift); + for (size_t i = as + extra_words; i >= extra_words + 2; i--) { + bitset->array[i - 1] = + (bitset->array[i - 1 - extra_words] << inword_shift) | + (bitset->array[i - 2 - extra_words] >> (64 - inword_shift)); + } + bitset->array[extra_words] = bitset->array[0] << inword_shift; + } + for (size_t i = 0; i < extra_words; i++) { + bitset->array[i] = 0; + } +} + +void bitset_shift_right(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + // could be done with a memmove + for (size_t i = 0; i < as - extra_words; i++) { + bitset->array[i] = bitset->array[i + extra_words]; + } + bitset_resize(bitset, as - extra_words, false); + + } else { + for (size_t i = 0; i + extra_words + 1 < as; i++) { + bitset->array[i] = + (bitset->array[i + extra_words] >> inword_shift) | + (bitset->array[i + extra_words + 1] << (64 - inword_shift)); + } + bitset->array[as - extra_words - 1] = + (bitset->array[as - 1] >> inword_shift); + bitset_resize(bitset, as - extra_words, false); + } +} + +/* Free memory. */ +void bitset_free(bitset_t *bitset) { + if (bitset == NULL) { + return; + } + roaring_free(bitset->array); + roaring_free(bitset); +} + +/* Resize the bitset so that it can support newarraysize * 64 bits. Return true + * in case of success, false for failure. */ +bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes) { + if (newarraysize > SIZE_MAX / 64) { + return false; + } + size_t smallest = + newarraysize < bitset->arraysize ? newarraysize : bitset->arraysize; + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = bitset->capacity; + if (newcapacity == 0) { + newcapacity = 1; + } + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; + } + if (padwithzeroes && (newarraysize > smallest)) + memset(bitset->array + smallest, 0, + sizeof(uint64_t) * (newarraysize - smallest)); + bitset->arraysize = newarraysize; + return true; // success! +} + +size_t bitset_count(const bitset_t *bitset) { + size_t card = 0; + size_t k = 0; + for (; k + 7 < bitset->arraysize; k += 8) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); + card += roaring_hamming(bitset->array[k + 4]); + card += roaring_hamming(bitset->array[k + 5]); + card += roaring_hamming(bitset->array[k + 6]); + card += roaring_hamming(bitset->array[k + 7]); + } + for (; k + 3 < bitset->arraysize; k += 4) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); + } + for (; k < bitset->arraysize; k++) { + card += roaring_hamming(bitset->array[k]); + } + return card; +} + +bool bitset_inplace_union(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + b1->array[k] |= b2->array[k]; + } + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; +} + +size_t bitset_minimum(const bitset_t *bitset) { + for (size_t k = 0; k < bitset->arraysize; k++) { + uint64_t w = bitset->array[k]; + if (w != 0) { + return roaring_trailing_zeroes(w) + k * 64; + } + } + return 0; +} + +bool bitset_grow(bitset_t *bitset, size_t newarraysize) { + if (newarraysize < bitset->arraysize) { + return false; + } + if (newarraysize > SIZE_MAX / 64) { + return false; + } + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = (UINT64_C(0xFFFFFFFFFFFFFFFF) >> + roaring_leading_zeroes(newarraysize)) + + 1; + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; + } + memset(bitset->array + bitset->arraysize, 0, + sizeof(uint64_t) * (newarraysize - bitset->arraysize)); + bitset->arraysize = newarraysize; + return true; // success! +} + +size_t bitset_maximum(const bitset_t *bitset) { + for (size_t k = bitset->arraysize; k > 0; k--) { + uint64_t w = bitset->array[k - 1]; + if (w != 0) { + return 63 - roaring_leading_zeroes(w) + (k - 1) * 64; + } + } + return 0; +} + +/* Returns true if bitsets share no common elements, false otherwise. + * + * Performs early-out if common element found. */ +bool bitsets_disjoint(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return false; + } + return true; +} + +/* Returns true if bitsets contain at least 1 common element, false if they are + * disjoint. + * + * Performs early-out if common element found. */ +bool bitsets_intersect(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return true; + } + return false; +} + +/* Returns true if b has any bits set in or after b->array[starting_loc]. */ +static bool any_bits_set(const bitset_t *b, size_t starting_loc) { + if (starting_loc >= b->arraysize) { + return false; + } + for (size_t k = starting_loc; k < b->arraysize; k++) { + if (b->array[k] != 0) return true; + } + return false; +} + +/* Returns true if b1 has all of b2's bits set. + * + * Performs early out if a bit is found in b2 that is not found in b1. */ +bool bitset_contains_all(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t min_size = b1->arraysize; + if (b1->arraysize > b2->arraysize) { + min_size = b2->arraysize; + } + for (size_t k = 0; k < min_size; k++) { + if ((b1->array[k] & b2->array[k]) != b2->array[k]) { + return false; + } + } + if (b2->arraysize > b1->arraysize) { + /* Need to check if b2 has any bits set beyond b1's array */ + return !any_bits_set(b2, b1->arraysize); + } + return true; +} + +size_t bitset_union_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k + 3 < minlength; k += 4) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + answer += roaring_hamming(b1->array[k + 1] | b2->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2] | b2->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3] | b2->array[k + 3]); + } + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + } + if (b2->arraysize > b1->arraysize) { + // k is equal to b1->arraysize + for (; k + 3 < b2->arraysize; k += 4) { + answer += roaring_hamming(b2->array[k]); + answer += roaring_hamming(b2->array[k + 1]); + answer += roaring_hamming(b2->array[k + 2]); + answer += roaring_hamming(b2->array[k + 3]); + } + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + // k is equal to b2->arraysize + for (; k + 3 < b1->arraysize; k += 4) { + answer += roaring_hamming(b1->array[k]); + answer += roaring_hamming(b1->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3]); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; +} + +void bitset_inplace_intersection(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= b2->array[k]; + } + for (; k < b1->arraysize; ++k) { + b1->array[k] = 0; // memset could, maybe, be a tiny bit faster + } +} + +size_t bitset_intersection_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & b2->array[k]); + } + return answer; +} + +void bitset_inplace_difference(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= ~(b2->array[k]); + } +} + +size_t bitset_difference_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & ~(b2->array[k])); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + return answer; +} + +bool bitset_inplace_symmetric_difference( + bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] ^= b2->array[k]; + } + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; +} + +size_t bitset_symmetric_difference_count( + const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] ^ b2->array[k]); + } + if (b2->arraysize > b1->arraysize) { + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; +} + +bool bitset_trim(bitset_t *bitset) { + size_t newsize = bitset->arraysize; + while (newsize > 0) { + if (bitset->array[newsize - 1] == 0) + newsize -= 1; + else + break; + } + if (bitset->capacity == newsize) return true; // nothing to do + uint64_t *newarray; + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newsize)) == NULL) { + return false; + } + bitset->array = newarray; + bitset->capacity = newsize; + bitset->arraysize = newsize; + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/bitset.c */ /* begin file src/bitset_util.c */ @@ -9301,12 +11488,18 @@ bool bitset_trim(bitset_t *bitset) { #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif - #ifdef __cplusplus using namespace ::roaring::internal; -extern "C" { namespace roaring { namespace api { +extern "C" { +namespace roaring { +namespace api { #endif #if CROARING_IS_X64 @@ -9854,29 +12047,34 @@ static uint16_t vecDecodeTable_uint16[256][8] = { #if CROARING_IS_X64 #if CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX512 -const uint8_t vbmi2_table[64] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63}; -size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, uint32_t *vout, - size_t outcapacity, uint32_t base) { +const uint8_t vbmi2_table[64] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}; +size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, + uint32_t *vout, size_t outcapacity, + uint32_t base) { uint32_t *out = (uint32_t *)vout; uint32_t *initout = out; uint32_t *safeout = out + outcapacity; - __m512i base_v = _mm512_set1_epi32(base); + __m512i base_v = _mm512_set1_epi32(base); __m512i index_table = _mm512_loadu_si512(vbmi2_table); size_t i = 0; - for (; (i < length) && ((out + 64) < safeout); i += 1) - { - uint64_t v = words[i]; - __m512i vec = _mm512_maskz_compress_epi8(v, index_table); - - uint8_t advance = roaring_hamming(v); - - __m512i vbase = _mm512_add_epi32(base_v, _mm512_set1_epi32(i * 64)); - __m512i r1 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec,0)); - __m512i r2 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec,1)); - __m512i r3 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec,2)); - __m512i r4 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec,3)); - + for (; (i < length) && ((out + 64) < safeout); i += 1) { + uint64_t v = words[i]; + __m512i vec = _mm512_maskz_compress_epi8(v, index_table); + + uint8_t advance = (uint8_t)roaring_hamming(v); + + __m512i vbase = + _mm512_add_epi32(base_v, _mm512_set1_epi32((int)(i * 64))); + __m512i r1 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 0)); + __m512i r2 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 1)); + __m512i r3 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 2)); + __m512i r4 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 3)); + r1 = _mm512_add_epi32(r1, vbase); r2 = _mm512_add_epi32(r2, vbase); r3 = _mm512_add_epi32(r3, vbase); @@ -9887,33 +12085,35 @@ size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, uint3 _mm512_storeu_si512((__m512i *)(out + 48), r4); out += advance; - } base += i * 64; - - for (; (i < length) && (out < safeout); ++i) { - uint64_t w = words[i]; - while ((w != 0) && (out < safeout)) { - uint64_t t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail) - int r = roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out, &val, - sizeof(uint32_t)); // should be compiled as a MOV on x64 - out++; - w ^= t; - } - base += 64; - } + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + out++; + w ^= t; + } + base += 64; + } return out - initout; - } -// Reference: https://lemire.me/blog/2022/05/10/faster-bitset-decoding-using-intel-avx-512/ -size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, size_t length, - uint16_t *vout, size_t capacity, uint16_t base) { +// Reference: +// https://lemire.me/blog/2022/05/10/faster-bitset-decoding-using-intel-avx-512/ +size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, + size_t length, uint16_t *vout, + size_t capacity, uint16_t base) { uint16_t *out = (uint16_t *)vout; uint16_t *initout = out; uint16_t *safeout = vout + capacity; @@ -9922,41 +12122,42 @@ size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, size_t length __m512i index_table = _mm512_loadu_si512(vbmi2_table); size_t i = 0; - for (; (i < length) && ((out + 64) < safeout); i++) - { + for (; (i < length) && ((out + 64) < safeout); i++) { uint64_t v = array[i]; __m512i vec = _mm512_maskz_compress_epi8(v, index_table); - uint8_t advance = roaring_hamming(v); + uint8_t advance = (uint8_t)roaring_hamming(v); - __m512i vbase = _mm512_add_epi16(base_v, _mm512_set1_epi16(i * 64)); - __m512i r1 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec,0)); - __m512i r2 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec,1)); + __m512i vbase = + _mm512_add_epi16(base_v, _mm512_set1_epi16((short)(i * 64))); + __m512i r1 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 0)); + __m512i r2 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 1)); r1 = _mm512_add_epi16(r1, vbase); r2 = _mm512_add_epi16(r2, vbase); - _mm512_storeu_si512((__m512i *)out, r1); + _mm512_storeu_si512((__m512i *)out, r1); _mm512_storeu_si512((__m512i *)(out + 32), r2); out += advance; - } base += i * 64; for (; (i < length) && (out < safeout); ++i) { - uint64_t w = array[i]; - while ((w != 0) && (out < safeout)) { - uint64_t t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail) - int r = roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out, &val, - sizeof(uint16_t)); - out++; - w ^= t; - } - base += 64; - } + uint64_t w = array[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, sizeof(uint16_t)); + out++; + w ^= t; + } + base += 64; + } return out - initout; } @@ -10003,8 +12204,11 @@ size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, for (; (i < length) && (out < safeout); ++i) { uint64_t w = words[i]; while ((w != 0) && (out < safeout)) { - uint64_t t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail) - int r = roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT uint32_t val = r + base; memcpy(out, &val, sizeof(uint32_t)); // should be compiled as a MOV on x64 @@ -10024,8 +12228,11 @@ size_t bitset_extract_setbits(const uint64_t *words, size_t length, for (size_t i = 0; i < length; ++i) { uint64_t w = words[i]; while (w != 0) { - uint64_t t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail) - int r = roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT uint32_t val = r + base; memcpy(out + outpos, &val, sizeof(uint32_t)); // should be compiled as a MOV on x64 @@ -10037,17 +12244,16 @@ size_t bitset_extract_setbits(const uint64_t *words, size_t length, return outpos; } -size_t bitset_extract_intersection_setbits_uint16(const uint64_t * __restrict__ words1, - const uint64_t * __restrict__ words2, - size_t length, uint16_t *out, - uint16_t base) { +size_t bitset_extract_intersection_setbits_uint16( + const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, + size_t length, uint16_t *out, uint16_t base) { int outpos = 0; for (size_t i = 0; i < length; ++i) { uint64_t w = words1[i] & words2[i]; while (w != 0) { uint64_t t = w & (~w + 1); int r = roaring_trailing_zeroes(w); - out[outpos++] = r + base; + out[outpos++] = (uint16_t)(r + base); w ^= t; } base += 64; @@ -10111,7 +12317,7 @@ size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, while ((w != 0) && (out < safeout)) { uint64_t t = w & (~w + 1); int r = roaring_trailing_zeroes(w); - *out = r + base; + *out = (uint16_t)(r + base); out++; w ^= t; } @@ -10139,7 +12345,7 @@ size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, while (w != 0) { uint64_t t = w & (~w + 1); int r = roaring_trailing_zeroes(w); - out[outpos++] = r + base; + out[outpos++] = (uint16_t)(r + base); w ^= t; } base += 64; @@ -10149,8 +12355,10 @@ size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, #if defined(CROARING_ASMBITMANIPOPTIMIZATION) && defined(CROARING_IS_X64) -static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, uint64_t card, - const uint16_t *list, uint64_t length) { +static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { uint64_t offset, load, pos; uint64_t shift = 6; const uint16_t *end = list + length; @@ -10174,7 +12382,8 @@ static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, uint64_t c return card; } -static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { +static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, + uint64_t length) { uint64_t pos; const uint16_t *end = list + length; @@ -10229,8 +12438,9 @@ static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, u } } -static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, - uint64_t length) { +static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { uint64_t offset, load, pos; uint64_t shift = 6; const uint16_t *end = list + length; @@ -10255,8 +12465,9 @@ static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, co return card; } -static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, - uint64_t length) { +static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { uint64_t offset, load, newload, pos, index; const uint16_t *end = list + length; while (list != end) { @@ -10272,8 +12483,10 @@ static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, return card; } -static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, uint64_t card, - const uint16_t *list, uint64_t length) { +static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { uint64_t offset, load, newload, pos, index; const uint16_t *end = list + length; while (list != end) { @@ -10289,7 +12502,9 @@ static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, uint64_ return card; } -static inline void _scalar_bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { +static inline void _scalar_bitset_set_list(uint64_t *words, + const uint16_t *list, + uint64_t length) { uint64_t offset, load, newload, pos, index; const uint16_t *end = list + length; while (list != end) { @@ -10305,7 +12520,7 @@ static inline void _scalar_bitset_set_list(uint64_t *words, const uint16_t *list uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, uint64_t length) { - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { return _asm_bitset_clear_list(words, card, list, length); } else { return _scalar_bitset_clear_list(words, card, list, length); @@ -10314,7 +12529,7 @@ uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, const uint16_t *list, uint64_t length) { - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { return _asm_bitset_set_list_withcard(words, card, list, length); } else { return _scalar_bitset_set_list_withcard(words, card, list, length); @@ -10322,7 +12537,7 @@ uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, } void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { _asm_bitset_set_list(words, list, length); } else { _scalar_bitset_set_list(words, list, length); @@ -10416,9 +12631,13 @@ void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length) { } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace api { +} +} +} // extern "C" { namespace roaring { namespace api { #endif -/* end file src/bitset_util.c */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif/* end file src/bitset_util.c */ /* begin file src/containers/array.c */ /* * array.c @@ -10429,28 +12648,38 @@ void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length) { #include #include + #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 #endif #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif extern inline uint16_t array_container_minimum(const array_container_t *arr); extern inline uint16_t array_container_maximum(const array_container_t *arr); -extern inline int array_container_index_equalorlarger(const array_container_t *arr, uint16_t x); +extern inline int array_container_index_equalorlarger( + const array_container_t *arr, uint16_t x); extern inline int array_container_rank(const array_container_t *arr, uint16_t x); +extern inline uint32_t array_container_rank_many(const array_container_t *arr, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, + uint64_t *ans); extern inline int array_container_get_index(const array_container_t *arr, - uint16_t x); + uint16_t x); extern inline bool array_container_contains(const array_container_t *arr, uint16_t pos); extern inline int array_container_cardinality(const array_container_t *array); -extern inline bool array_container_nonzero_cardinality(const array_container_t *array); +extern inline bool array_container_nonzero_cardinality( + const array_container_t *array); extern inline int32_t array_container_serialized_size_in_bytes(int32_t card); extern inline bool array_container_empty(const array_container_t *array); extern inline bool array_container_full(const array_container_t *array); @@ -10459,15 +12688,15 @@ extern inline bool array_container_full(const array_container_t *array); array_container_t *array_container_create_given_capacity(int32_t size) { array_container_t *container; - if ((container = (array_container_t *)roaring_malloc(sizeof(array_container_t))) == - NULL) { + if ((container = (array_container_t *)roaring_malloc( + sizeof(array_container_t))) == NULL) { return NULL; } - if( size <= 0 ) { // we don't want to rely on malloc(0) + if (size <= 0) { // we don't want to rely on malloc(0) container->array = NULL; - } else if ((container->array = (uint16_t *)roaring_malloc(sizeof(uint16_t) * size)) == - NULL) { + } else if ((container->array = (uint16_t *)roaring_malloc(sizeof(uint16_t) * + size)) == NULL) { roaring_free(container); return NULL; } @@ -10484,17 +12713,19 @@ array_container_t *array_container_create(void) { } /* Create a new array containing all values in [min,max). */ -array_container_t * array_container_create_range(uint32_t min, uint32_t max) { - array_container_t * answer = array_container_create_given_capacity(max - min + 1); - if(answer == NULL) return answer; +array_container_t *array_container_create_range(uint32_t min, uint32_t max) { + array_container_t *answer = + array_container_create_given_capacity(max - min + 1); + if (answer == NULL) return answer; answer->cardinality = 0; - for(uint32_t k = min; k < max; k++) { - answer->array[answer->cardinality++] = k; + for (uint32_t k = min; k < max; k++) { + answer->array[answer->cardinality++] = k; } return answer; } /* Duplicate container */ +ALLOW_UNALIGNED array_container_t *array_container_clone(const array_container_t *src) { array_container_t *newcontainer = array_container_create_given_capacity(src->capacity); @@ -10508,9 +12739,8 @@ array_container_t *array_container_clone(const array_container_t *src) { return newcontainer; } -void array_container_offset(const array_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset) { +void array_container_offset(const array_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { array_container_t *lo = NULL, *hi = NULL; int top, lo_cap, hi_cap; @@ -10522,7 +12752,7 @@ void array_container_offset(const array_container_t *c, for (int i = 0; i < lo_cap; ++i) { array_container_add(lo, c->array[i] + offset); } - *loc = (container_t*)lo; + *loc = (container_t *)lo; } hi_cap = c->cardinality - lo_cap; @@ -10531,7 +12761,7 @@ void array_container_offset(const array_container_t *c, for (int i = lo_cap; i < c->cardinality; ++i) { array_container_add(hi, c->array[i] + offset); } - *hic = (container_t*)hi; + *hic = (container_t *)hi; } } @@ -10539,32 +12769,34 @@ int array_container_shrink_to_fit(array_container_t *src) { if (src->cardinality == src->capacity) return 0; // nothing to do int savings = src->capacity - src->cardinality; src->capacity = src->cardinality; - if( src->capacity == 0) { // we do not want to rely on realloc for zero allocs - roaring_free(src->array); - src->array = NULL; + if (src->capacity == + 0) { // we do not want to rely on realloc for zero allocs + roaring_free(src->array); + src->array = NULL; } else { - uint16_t *oldarray = src->array; - src->array = - (uint16_t *)roaring_realloc(oldarray, src->capacity * sizeof(uint16_t)); - if (src->array == NULL) roaring_free(oldarray); // should never happen? + uint16_t *oldarray = src->array; + src->array = (uint16_t *)roaring_realloc( + oldarray, src->capacity * sizeof(uint16_t)); + if (src->array == NULL) roaring_free(oldarray); // should never happen? } return savings; } /* Free memory. */ void array_container_free(array_container_t *arr) { - if(arr->array != NULL) {// Jon Strabala reports that some tools complain otherwise + if (arr->array != + NULL) { // Jon Strabala reports that some tools complain otherwise roaring_free(arr->array); - arr->array = NULL; // pedantic + arr->array = NULL; // pedantic } roaring_free(arr); } static inline int32_t grow_capacity(int32_t capacity) { - return (capacity <= 0) ? ARRAY_DEFAULT_INIT_SIZE - : capacity < 64 ? capacity * 2 - : capacity < 1024 ? capacity * 3 / 2 - : capacity * 5 / 4; + return (capacity <= 0) ? ARRAY_DEFAULT_INIT_SIZE + : capacity < 64 ? capacity * 2 + : capacity < 1024 ? capacity * 3 / 2 + : capacity * 5 / 4; } static inline int32_t clamp(int32_t val, int32_t min, int32_t max) { @@ -10573,7 +12805,6 @@ static inline int32_t clamp(int32_t val, int32_t min, int32_t max) { void array_container_grow(array_container_t *container, int32_t min, bool preserve) { - int32_t max = (min <= DEFAULT_MAX_SIZE ? DEFAULT_MAX_SIZE : 65536); int32_t new_capacity = clamp(grow_capacity(container->capacity), min, max); @@ -10587,9 +12818,10 @@ void array_container_grow(array_container_t *container, int32_t min, } else { // Jon Strabala reports that some tools complain otherwise if (array != NULL) { - roaring_free(array); + roaring_free(array); } - container->array = (uint16_t *)roaring_malloc(new_capacity * sizeof(uint16_t)); + container->array = + (uint16_t *)roaring_malloc(new_capacity * sizeof(uint16_t)); } // if realloc fails, we have container->array == NULL. @@ -10624,11 +12856,10 @@ void array_container_union(const array_container_t *array_1, const int32_t max_cardinality = card_1 + card_2; if (out->capacity < max_cardinality) { - array_container_grow(out, max_cardinality, false); + array_container_grow(out, max_cardinality, false); } - out->cardinality = (int32_t)fast_union_uint16(array_1->array, card_1, - array_2->array, card_2, out->array); - + out->cardinality = (int32_t)fast_union_uint16( + array_1->array, card_1, array_2->array, card_2, out->array); } /* Computes the difference of array1 and array2 and write the result @@ -10641,15 +12872,16 @@ void array_container_andnot(const array_container_t *array_1, if (out->capacity < array_1->cardinality) array_container_grow(out, array_1->cardinality, false); #if CROARING_IS_X64 - if(( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) && (out != array_1) && (out != array_2)) { - out->cardinality = - difference_vector16(array_1->array, array_1->cardinality, - array_2->array, array_2->cardinality, out->array); - } else { - out->cardinality = - difference_uint16(array_1->array, array_1->cardinality, array_2->array, - array_2->cardinality, out->array); - } + if ((croaring_hardware_support() & ROARING_SUPPORTS_AVX2) && + (out != array_1) && (out != array_2)) { + out->cardinality = difference_vector16( + array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); + } else { + out->cardinality = + difference_uint16(array_1->array, array_1->cardinality, + array_2->array, array_2->cardinality, out->array); + } #else out->cardinality = difference_uint16(array_1->array, array_1->cardinality, array_2->array, @@ -10672,14 +12904,14 @@ void array_container_xor(const array_container_t *array_1, } #if CROARING_IS_X64 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { - out->cardinality = - xor_vector16(array_1->array, array_1->cardinality, array_2->array, - array_2->cardinality, out->array); + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + out->cardinality = + xor_vector16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); } else { - out->cardinality = - xor_uint16(array_1->array, array_1->cardinality, array_2->array, - array_2->cardinality, out->array); + out->cardinality = + xor_uint16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); } #else out->cardinality = @@ -10704,12 +12936,12 @@ void array_container_intersection(const array_container_t *array1, const int threshold = 64; // subject to tuning #if CROARING_IS_X64 if (out->capacity < min_card) { - array_container_grow(out, min_card + sizeof(__m128i) / sizeof(uint16_t), - false); + array_container_grow(out, min_card + sizeof(__m128i) / sizeof(uint16_t), + false); } #else if (out->capacity < min_card) { - array_container_grow(out, min_card, false); + array_container_grow(out, min_card, false); } #endif @@ -10721,13 +12953,13 @@ void array_container_intersection(const array_container_t *array1, array2->array, card_2, array1->array, card_1, out->array); } else { #if CROARING_IS_X64 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { - out->cardinality = intersect_vector16( - array1->array, card_1, array2->array, card_2, out->array); - } else { - out->cardinality = intersect_uint16(array1->array, card_1, - array2->array, card_2, out->array); - } + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + out->cardinality = intersect_vector16( + array1->array, card_1, array2->array, card_2, out->array); + } else { + out->cardinality = intersect_uint16( + array1->array, card_1, array2->array, card_2, out->array); + } #else out->cardinality = intersect_uint16(array1->array, card_1, array2->array, card_2, out->array); @@ -10749,13 +12981,13 @@ int array_container_intersection_cardinality(const array_container_t *array1, array1->array, card_1); } else { #if CROARING_IS_X64 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { - return intersect_vector16_cardinality(array1->array, card_1, - array2->array, card_2); - } else { - return intersect_uint16_cardinality(array1->array, card_1, - array2->array, card_2); - } + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return intersect_vector16_cardinality(array1->array, card_1, + array2->array, card_2); + } else { + return intersect_uint16_cardinality(array1->array, card_1, + array2->array, card_2); + } #else return intersect_uint16_cardinality(array1->array, card_1, array2->array, card_2); @@ -10764,19 +12996,19 @@ int array_container_intersection_cardinality(const array_container_t *array1, } bool array_container_intersect(const array_container_t *array1, - const array_container_t *array2) { + const array_container_t *array2) { int32_t card_1 = array1->cardinality, card_2 = array2->cardinality; const int threshold = 64; // subject to tuning if (card_1 * threshold < card_2) { - return intersect_skewed_uint16_nonempty( - array1->array, card_1, array2->array, card_2); + return intersect_skewed_uint16_nonempty(array1->array, card_1, + array2->array, card_2); } else if (card_2 * threshold < card_1) { - return intersect_skewed_uint16_nonempty( - array2->array, card_2, array1->array, card_1); + return intersect_skewed_uint16_nonempty(array2->array, card_2, + array1->array, card_1); } else { - // we do not bother vectorizing - return intersect_uint16_nonempty(array1->array, card_1, - array2->array, card_2); + // we do not bother vectorizing + return intersect_uint16_nonempty(array1->array, card_1, array2->array, + card_2); } } @@ -10804,7 +13036,7 @@ void array_container_intersection_inplace(array_container_t *src_1, } #else src_1->cardinality = intersect_uint16( - src_1->array, card_1, src_2->array, card_2, src_1->array); + src_1->array, card_1, src_2->array, card_2, src_1->array); #endif } } @@ -10812,22 +13044,23 @@ void array_container_intersection_inplace(array_container_t *src_1, ALLOW_UNALIGNED int array_container_to_uint32_array(void *vout, const array_container_t *cont, uint32_t base) { - #if CROARING_IS_X64 int support = croaring_hardware_support(); #if CROARING_COMPILER_SUPPORTS_AVX512 if (support & ROARING_SUPPORTS_AVX512) { - return avx512_array_container_to_uint32_array(vout, cont->array, cont->cardinality, base); + return avx512_array_container_to_uint32_array(vout, cont->array, + cont->cardinality, base); } #endif if (support & ROARING_SUPPORTS_AVX2) { - return array_container_to_uint32_array_vector16(vout, cont->array, cont->cardinality, base); + return array_container_to_uint32_array_vector16( + vout, cont->array, cont->cardinality, base); } -#endif // CROARING_IS_X64 +#endif // CROARING_IS_X64 int outpos = 0; uint32_t *out = (uint32_t *)vout; size_t i = 0; - for ( ; i < (size_t)cont->cardinality; ++i) { + for (; i < (size_t)cont->cardinality; ++i) { const uint32_t val = base + cont->array[i]; memcpy(out + outpos, &val, sizeof(uint32_t)); // should be compiled as a MOV on x64 @@ -10881,7 +13114,8 @@ bool array_container_validate(const array_container_t *v, const char **reason) { return false; } if (v->cardinality == 0) { - return true; + *reason = "zero cardinality"; + return false; } if (v->array == NULL) { @@ -10974,7 +13208,9 @@ bool array_container_iterate64(const array_container_t *cont, uint32_t base, } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/array.c */ /* begin file src/containers/bitset.c */ @@ -10994,22 +13230,34 @@ bool array_container_iterate64(const array_container_t *cont, uint32_t base, #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 #endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif -extern inline int bitset_container_cardinality(const bitset_container_t *bitset); -extern inline void bitset_container_set(bitset_container_t *bitset, uint16_t pos); +extern inline int bitset_container_cardinality( + const bitset_container_t *bitset); +extern inline void bitset_container_set(bitset_container_t *bitset, + uint16_t pos); // unused at this time: -//extern inline void bitset_container_unset(bitset_container_t *bitset, uint16_t pos); +// extern inline void bitset_container_unset(bitset_container_t *bitset, +// uint16_t pos); extern inline bool bitset_container_get(const bitset_container_t *bitset, uint16_t pos); extern inline int32_t bitset_container_serialized_size_in_bytes(void); -extern inline bool bitset_container_add(bitset_container_t *bitset, uint16_t pos); -extern inline bool bitset_container_remove(bitset_container_t *bitset, uint16_t pos); +extern inline bool bitset_container_add(bitset_container_t *bitset, + uint16_t pos); +extern inline bool bitset_container_remove(bitset_container_t *bitset, + uint16_t pos); extern inline bool bitset_container_contains(const bitset_container_t *bitset, uint16_t pos); @@ -11024,8 +13272,6 @@ void bitset_container_set_all(bitset_container_t *bitset) { bitset->cardinality = (1 << 16); } - - /* Create a new bitset. Return NULL in case of failure. */ bitset_container_t *bitset_container_create(void) { bitset_container_t *bitset = @@ -11038,13 +13284,12 @@ bitset_container_t *bitset_container_create(void) { size_t align_size = 32; #if CROARING_IS_X64 int support = croaring_hardware_support(); - if ( support & ROARING_SUPPORTS_AVX512 ) { - // sizeof(__m512i) == 64 - align_size = 64; - } - else { - // sizeof(__m256i) == 32 - align_size = 32; + if (support & ROARING_SUPPORTS_AVX512) { + // sizeof(__m512i) == 64 + align_size = 64; + } else { + // sizeof(__m256i) == 32 + align_size = 32; } #endif bitset->words = (uint64_t *)roaring_aligned_malloc( @@ -11095,14 +13340,16 @@ void bitset_container_add_from_range(bitset_container_t *bitset, uint32_t min, /* Free memory. */ void bitset_container_free(bitset_container_t *bitset) { - if(bitset->words != NULL) {// Jon Strabala reports that some tools complain otherwise - roaring_aligned_free(bitset->words); - bitset->words = NULL; // pedantic + if (bitset->words != + NULL) { // Jon Strabala reports that some tools complain otherwise + roaring_aligned_free(bitset->words); + bitset->words = NULL; // pedantic } roaring_free(bitset); } /* duplicate container. */ +ALLOW_UNALIGNED bitset_container_t *bitset_container_clone(const bitset_container_t *src) { bitset_container_t *bitset = (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); @@ -11112,14 +13359,13 @@ bitset_container_t *bitset_container_clone(const bitset_container_t *src) { } size_t align_size = 32; -#if CROARING_IS_X64 - if ( croaring_hardware_support() & ROARING_SUPPORTS_AVX512 ) { - // sizeof(__m512i) == 64 - align_size = 64; - } - else { - // sizeof(__m256i) == 32 - align_size = 32; +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + // sizeof(__m512i) == 64 + align_size = 64; + } else { + // sizeof(__m256i) == 32 + align_size = 32; } #endif bitset->words = (uint64_t *)roaring_aligned_malloc( @@ -11134,9 +13380,8 @@ bitset_container_t *bitset_container_clone(const bitset_container_t *src) { return bitset; } -void bitset_container_offset(const bitset_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset) { +void bitset_container_offset(const bitset_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { bitset_container_t *bc = NULL; uint64_t val; uint16_t b, i, end; @@ -11148,13 +13393,13 @@ void bitset_container_offset(const bitset_container_t *c, if (loc != NULL) { bc = bitset_container_create(); if (i == 0) { - memcpy(bc->words+b, c->words, 8*end); + memcpy(bc->words + b, c->words, 8 * end); } else { bc->words[b] = c->words[0] << i; for (uint32_t k = 1; k < end; ++k) { val = c->words[k] << i; - val |= c->words[k-1] >> (64 - i); - bc->words[b+k] = val; + val |= c->words[k - 1] >> (64 - i); + bc->words[b + k] = val; } } @@ -11171,7 +13416,7 @@ void bitset_container_offset(const bitset_container_t *c, // Both hic and loc can't be NULL, so bc is never NULL here if (bc->cardinality == 0) { bitset_container_free(bc); - } + } return; } @@ -11180,20 +13425,20 @@ void bitset_container_offset(const bitset_container_t *c, } if (i == 0) { - memcpy(bc->words, c->words+end, 8*b); + memcpy(bc->words, c->words + end, 8 * b); } else { for (uint32_t k = end; k < 1024; ++k) { val = c->words[k] << i; - val |= c->words[k-1] >> (64 - i); - bc->words[k-end] = val; + val |= c->words[k - 1] >> (64 - i); + bc->words[k - end] = val; } bc->words[b] = c->words[1023] >> (64 - i); } bc->cardinality = bitset_container_compute_cardinality(bc); if (bc->cardinality == 0) { - bitset_container_free(bc); - return; + bitset_container_free(bc); + return; } *hic = bc; } @@ -11205,56 +13450,54 @@ void bitset_container_set_range(bitset_container_t *bitset, uint32_t begin, bitset_container_compute_cardinality(bitset); // could be smarter } - bool bitset_container_intersect(const bitset_container_t *src_1, - const bitset_container_t *src_2) { - // could vectorize, but this is probably already quite fast in practice - const uint64_t * __restrict__ words_1 = src_1->words; - const uint64_t * __restrict__ words_2 = src_2->words; - for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i ++) { - if((words_1[i] & words_2[i]) != 0) return true; + const bitset_container_t *src_2) { + // could vectorize, but this is probably already quite fast in practice + const uint64_t *__restrict__ words_1 = src_1->words; + const uint64_t *__restrict__ words_2 = src_2->words; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { + if ((words_1[i] & words_2[i]) != 0) return true; } return false; } - #if CROARING_IS_X64 -#ifndef WORDS_IN_AVX2_REG -#define WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) +#ifndef CROARING_WORDS_IN_AVX2_REG +#define CROARING_WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) #endif #ifndef WORDS_IN_AVX512_REG #define WORDS_IN_AVX512_REG sizeof(__m512i) / sizeof(uint64_t) #endif /* Get the number of bits set (force computation) */ -static inline int _scalar_bitset_container_compute_cardinality(const bitset_container_t *bitset) { - const uint64_t *words = bitset->words; - int32_t sum = 0; - for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 4) { - sum += roaring_hamming(words[i]); - sum += roaring_hamming(words[i + 1]); - sum += roaring_hamming(words[i + 2]); - sum += roaring_hamming(words[i + 3]); - } - return sum; +static inline int _scalar_bitset_container_compute_cardinality( + const bitset_container_t *bitset) { + const uint64_t *words = bitset->words; + int32_t sum = 0; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 4) { + sum += roaring_hamming(words[i]); + sum += roaring_hamming(words[i + 1]); + sum += roaring_hamming(words[i + 2]); + sum += roaring_hamming(words[i + 3]); + } + return sum; } /* Get the number of bits set (force computation) */ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { int support = croaring_hardware_support(); #if CROARING_COMPILER_SUPPORTS_AVX512 - if( support & ROARING_SUPPORTS_AVX512 ) { - return (int) avx512_vpopcount( - (const __m512i *)bitset->words, - BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG)); + if (support & ROARING_SUPPORTS_AVX512) { + return (int)avx512_vpopcount( + (const __m512i *)bitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG)); } else -#endif // CROARING_COMPILER_SUPPORTS_AVX512 - if( support & ROARING_SUPPORTS_AVX2 ) { - return (int) avx2_harley_seal_popcount256( - (const __m256i *)bitset->words, - BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); - } else { - return _scalar_bitset_container_compute_cardinality(bitset); - - } +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { + return (int)avx2_harley_seal_popcount256( + (const __m256i *)bitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS / (CROARING_WORDS_IN_AVX2_REG)); + } else { + return _scalar_bitset_container_compute_cardinality(bitset); + } } #elif defined(CROARING_USENEON) @@ -11281,7 +13524,7 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { return vgetq_lane_u64(n, 0) + vgetq_lane_u64(n, 1); } -#else // CROARING_IS_X64 +#else // CROARING_IS_X64 /* Get the number of bits set (force computation) */ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { @@ -11296,19 +13539,19 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { return sum; } -#endif // CROARING_IS_X64 +#endif // CROARING_IS_X64 #if CROARING_IS_X64 -#define BITSET_CONTAINER_FN_REPEAT 8 +#define CROARING_BITSET_CONTAINER_FN_REPEAT 8 #ifndef WORDS_IN_AVX512_REG #define WORDS_IN_AVX512_REG sizeof(__m512i) / sizeof(uint64_t) -#endif // WORDS_IN_AVX512_REG +#endif // WORDS_IN_AVX512_REG /* Computes a binary operation (eg union) on bitset1 and bitset2 and write the result to bitsetout */ // clang-format off -#define AVX512_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX512_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ static inline int _avx512_bitset_container_##opname##_nocard( \ const bitset_container_t *src_1, const bitset_container_t *src_2, \ @@ -11362,7 +13605,7 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { return dst->cardinality; \ } -#define AVX512_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX512_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ /* next, a version that updates cardinality*/ \ static inline int _avx512_bitset_container_##opname(const bitset_container_t *src_1, \ @@ -11376,7 +13619,7 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { return dst->cardinality; \ } -#define AVX512_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX512_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ /* next, a version that just computes the cardinality*/ \ static inline int _avx512_bitset_container_##opname##_justcard( \ @@ -11391,85 +13634,85 @@ int bitset_container_compute_cardinality(const bitset_container_t *bitset) { // we duplicate the function because other containers use the "or" term, makes API more consistent #if CROARING_COMPILER_SUPPORTS_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 // we duplicate the function because other containers use the "or" term, makes API more consistent CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 // we duplicate the function because other containers use the "or" term, makes API more consistent CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 CROARING_TARGET_AVX512 -AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) CROARING_UNTARGET_AVX512 #endif // CROARING_COMPILER_SUPPORTS_AVX512 -#ifndef WORDS_IN_AVX2_REG -#define WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) -#endif // WORDS_IN_AVX2_REG -#define LOOP_SIZE \ +#ifndef CROARING_WORDS_IN_AVX2_REG +#define CROARING_WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) +#endif // CROARING_WORDS_IN_AVX2_REG +#define CROARING_LOOP_SIZE \ BITSET_CONTAINER_SIZE_IN_WORDS / \ - ((WORDS_IN_AVX2_REG)*BITSET_CONTAINER_FN_REPEAT) + ((CROARING_WORDS_IN_AVX2_REG)*CROARING_BITSET_CONTAINER_FN_REPEAT) /* Computes a binary operation (eg union) on bitset1 and bitset2 and write the result to bitsetout */ // clang-format off -#define AVX_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ static inline int _avx2_bitset_container_##opname##_nocard( \ const bitset_container_t *src_1, const bitset_container_t *src_2, \ @@ -11480,7 +13723,7 @@ CROARING_UNTARGET_AVX512 uint8_t *out = (uint8_t *)dst->words; \ const int innerloop = 8; \ for (size_t i = 0; \ - i < BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG); \ + i < BITSET_CONTAINER_SIZE_IN_WORDS / (CROARING_WORDS_IN_AVX2_REG); \ i += innerloop) { \ __m256i A1, A2, AO; \ A1 = _mm256_lddqu_si256((const __m256i *)(words_1)); \ @@ -11523,7 +13766,7 @@ CROARING_UNTARGET_AVX512 return dst->cardinality; \ } -#define AVX_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ /* next, a version that updates cardinality*/ \ static inline int _avx2_bitset_container_##opname(const bitset_container_t *src_1, \ @@ -11534,11 +13777,11 @@ CROARING_UNTARGET_AVX512 __m256i *out = (__m256i *)dst->words; \ dst->cardinality = (int32_t)avx2_harley_seal_popcount256andstore_##opname( \ words_2, words_1, out, \ - BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); \ + BITSET_CONTAINER_SIZE_IN_WORDS / (CROARING_WORDS_IN_AVX2_REG)); \ return dst->cardinality; \ } \ -#define AVX_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ +#define CROARING_AVX_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ neon_intrinsic, after) \ /* next, a version that just computes the cardinality*/ \ static inline int _avx2_bitset_container_##opname##_justcard( \ @@ -11546,77 +13789,77 @@ CROARING_UNTARGET_AVX512 const __m256i *__restrict__ data1 = (const __m256i *)src_1->words; \ const __m256i *__restrict__ data2 = (const __m256i *)src_2->words; \ return (int)avx2_harley_seal_popcount256_##opname( \ - data2, data1, BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); \ + data2, data1, BITSET_CONTAINER_SIZE_IN_WORDS / (CROARING_WORDS_IN_AVX2_REG)); \ } // we duplicate the function because other containers use the "or" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 // we duplicate the function because other containers use the "or" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 // we duplicate the function because other containers use the "or" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 // we duplicate the function because other containers use the "intersection" term, makes API more consistent CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) CROARING_UNTARGET_AVX2 @@ -11678,7 +13921,7 @@ SCALAR_BITSET_CONTAINER_FN(xor, ^, _mm256_xor_si256, veorq_u64) SCALAR_BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) #if CROARING_COMPILER_SUPPORTS_AVX512 -#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +#define CROARING_BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ int bitset_container_##opname(const bitset_container_t *src_1, \ const bitset_container_t *src_2, \ bitset_container_t *dst) { \ @@ -11721,7 +13964,7 @@ SCALAR_BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) #else // CROARING_COMPILER_SUPPORTS_AVX512 -#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +#define CROARING_BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ int bitset_container_##opname(const bitset_container_t *src_1, \ const bitset_container_t *src_2, \ bitset_container_t *dst) { \ @@ -11753,7 +13996,7 @@ SCALAR_BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) #elif defined(CROARING_USENEON) -#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +#define CROARING_BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ int bitset_container_##opname(const bitset_container_t *src_1, \ const bitset_container_t *src_2, \ bitset_container_t *dst) { \ @@ -11841,7 +14084,7 @@ int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ #else -#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +#define CROARING_BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ int bitset_container_##opname(const bitset_container_t *src_1, \ const bitset_container_t *src_2, \ bitset_container_t *dst) { \ @@ -11889,15 +14132,15 @@ int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ #endif // CROARING_IS_X64 // we duplicate the function because other containers use the "or" term, makes API more consistent -BITSET_CONTAINER_FN(or, |, _mm256_or_si256, vorrq_u64) -BITSET_CONTAINER_FN(union, |, _mm256_or_si256, vorrq_u64) +CROARING_BITSET_CONTAINER_FN(or, |, _mm256_or_si256, vorrq_u64) +CROARING_BITSET_CONTAINER_FN(union, |, _mm256_or_si256, vorrq_u64) // we duplicate the function because other containers use the "intersection" term, makes API more consistent -BITSET_CONTAINER_FN(and, &, _mm256_and_si256, vandq_u64) -BITSET_CONTAINER_FN(intersection, &, _mm256_and_si256, vandq_u64) +CROARING_BITSET_CONTAINER_FN(and, &, _mm256_and_si256, vandq_u64) +CROARING_BITSET_CONTAINER_FN(intersection, &, _mm256_and_si256, vandq_u64) -BITSET_CONTAINER_FN(xor, ^, _mm256_xor_si256, veorq_u64) -BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) +CROARING_BITSET_CONTAINER_FN(xor, ^, _mm256_xor_si256, veorq_u64) +CROARING_BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) // clang-format On @@ -11987,6 +14230,10 @@ bool bitset_container_validate(const bitset_container_t *v, const char **reason) *reason = "cardinality is incorrect"; return false; } + if (v->cardinality <= DEFAULT_MAX_SIZE) { + *reason = "cardinality is too small for a bitmap container"; + return false; + } // Attempt to forcibly load the first and last words, hopefully causing // a segfault or an address sanitizer error if words is not allocated. volatile uint64_t *words = v->words; @@ -12207,6 +14454,29 @@ int bitset_container_rank(const bitset_container_t *container, uint16_t x) { return sum; } +uint32_t bitset_container_rank_many(const bitset_container_t *container, uint64_t start_rank, const uint32_t* begin, const uint32_t* end, uint64_t* ans){ + const uint16_t high = (uint16_t)((*begin) >> 16); + int i = 0; + int sum = 0; + const uint32_t* iter = begin; + for(; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if(xhigh != high) return iter - begin; // stop at next container + + uint16_t xlow = (uint16_t)x; + for(int count = xlow / 64; i < count; i++){ + sum += roaring_hamming(container->words[i]); + } + uint64_t lastword = container->words[i]; + uint64_t lastpos = UINT64_C(1) << (xlow % 64); + uint64_t mask = lastpos + lastpos - 1; // smear right + *(ans++) = start_rank + sum + roaring_hamming(lastword & mask); + } + return iter - begin; +} + + /* Returns the index of x , if not exsist return -1 */ int bitset_container_get_index(const bitset_container_t *container, uint16_t x) { if (bitset_container_get(container, x)) { @@ -12244,42 +14514,53 @@ int bitset_container_index_equalorlarger(const bitset_container_t *container, ui #ifdef __cplusplus } } } // extern "C" { namespace roaring { namespace internal { #endif -/* end file src/containers/bitset.c */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif/* end file src/containers/bitset.c */ /* begin file src/containers/containers.c */ #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +// In Windows MSVC C++ compiler, (type){init} does not compile, +// it causes C4576: a parenthesized type followed by an initializer list is a +// non-standard explicit type conversion syntax The correct syntax is type{init} +#define ROARING_INIT_ROARING_CONTAINER_ITERATOR_T roaring_container_iterator_t +namespace roaring { +namespace internal { +#else +#define ROARING_INIT_ROARING_CONTAINER_ITERATOR_T (roaring_container_iterator_t) #endif +static inline uint32_t minimum_uint32(uint32_t a, uint32_t b) { + return (a < b) ? a : b; +} + extern inline const container_t *container_unwrap_shared( - const container_t *candidate_shared_container, uint8_t *type); + const container_t *candidate_shared_container, uint8_t *type); extern inline container_t *container_mutable_unwrap_shared( - container_t *candidate_shared_container, uint8_t *type); + container_t *candidate_shared_container, uint8_t *type); -extern inline int container_get_cardinality( - const container_t *c, uint8_t typecode); +extern inline int container_get_cardinality(const container_t *c, + uint8_t typecode); -extern inline container_t *container_iand( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_iand(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -extern inline container_t *container_ior( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_ior(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -extern inline container_t *container_ixor( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_ixor(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -extern inline container_t *container_iandnot( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_iandnot(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); void container_free(container_t *c, uint8_t type) { switch (type) { @@ -12318,38 +14599,35 @@ void container_printf(const container_t *c, uint8_t type) { } } -void container_printf_as_uint32_array( - const container_t *c, uint8_t typecode, - uint32_t base -){ +void container_printf_as_uint32_array(const container_t *c, uint8_t typecode, + uint32_t base) { c = container_unwrap_shared(c, &typecode); switch (typecode) { case BITSET_CONTAINER_TYPE: - bitset_container_printf_as_uint32_array( - const_CAST_bitset(c), base); + bitset_container_printf_as_uint32_array(const_CAST_bitset(c), base); return; case ARRAY_CONTAINER_TYPE: - array_container_printf_as_uint32_array( - const_CAST_array(c), base); + array_container_printf_as_uint32_array(const_CAST_array(c), base); return; case RUN_CONTAINER_TYPE: - run_container_printf_as_uint32_array( - const_CAST_run(c), base); + run_container_printf_as_uint32_array(const_CAST_run(c), base); return; default: roaring_unreachable; } } -bool container_internal_validate(const container_t *container, - uint8_t typecode, const char **reason) { +bool container_internal_validate(const container_t *container, uint8_t typecode, + const char **reason) { if (container == NULL) { *reason = "container is NULL"; return false; } - // Not using container_unwrap_shared because it asserts if shared containers are nested + // Not using container_unwrap_shared because it asserts if shared containers + // are nested if (typecode == SHARED_CONTAINER_TYPE) { - const shared_container_t *shared_container = const_CAST_shared(container); + const shared_container_t *shared_container = + const_CAST_shared(container); if (croaring_refcount_get(&shared_container->counter) == 0) { *reason = "shared container has zero refcount"; return false; @@ -12367,9 +14645,11 @@ bool container_internal_validate(const container_t *container, } switch (typecode) { case BITSET_CONTAINER_TYPE: - return bitset_container_validate(const_CAST_bitset(container), reason); + return bitset_container_validate(const_CAST_bitset(container), + reason); case ARRAY_CONTAINER_TYPE: - return array_container_validate(const_CAST_array(container), reason); + return array_container_validate(const_CAST_array(container), + reason); case RUN_CONTAINER_TYPE: return run_container_validate(const_CAST_run(container), reason); default: @@ -12378,44 +14658,34 @@ bool container_internal_validate(const container_t *container, } } -extern inline bool container_nonzero_cardinality( - const container_t *c, uint8_t typecode); +extern inline bool container_nonzero_cardinality(const container_t *c, + uint8_t typecode); -extern inline int container_to_uint32_array( - uint32_t *output, - const container_t *c, uint8_t typecode, - uint32_t base); +extern inline int container_to_uint32_array(uint32_t *output, + const container_t *c, + uint8_t typecode, uint32_t base); -extern inline container_t *container_add( - container_t *c, - uint16_t val, - uint8_t typecode, // !!! 2nd arg? - uint8_t *new_typecode); +extern inline container_t *container_add(container_t *c, uint16_t val, + uint8_t typecode, // !!! 2nd arg? + uint8_t *new_typecode); -extern inline bool container_contains( - const container_t *c, - uint16_t val, - uint8_t typecode); // !!! 2nd arg? +extern inline bool container_contains(const container_t *c, uint16_t val, + uint8_t typecode); // !!! 2nd arg? -extern inline container_t *container_and( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_and(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -extern inline container_t *container_or( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_or(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -extern inline container_t *container_xor( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_xor(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); -container_t *get_copy_of_container( - container_t *c, uint8_t *typecode, - bool copy_on_write -){ +container_t *get_copy_of_container(container_t *c, uint8_t *typecode, + bool copy_on_write) { if (copy_on_write) { shared_container_t *shared_container; if (*typecode == SHARED_CONTAINER_TYPE) { @@ -12462,7 +14732,8 @@ container_t *container_clone(const container_t *c, uint8_t typecode) { case RUN_CONTAINER_TYPE: return run_container_clone(const_CAST_run(c)); case SHARED_CONTAINER_TYPE: - // Shared containers are not cloneable. Are you mixing COW and non-COW bitmaps? + // Shared containers are not cloneable. Are you mixing COW and + // non-COW bitmaps? return NULL; default: assert(false); @@ -12471,9 +14742,8 @@ container_t *container_clone(const container_t *c, uint8_t typecode) { } } -container_t *shared_container_extract_copy( - shared_container_t *sc, uint8_t *typecode -){ +container_t *shared_container_extract_copy(shared_container_t *sc, + uint8_t *typecode) { assert(sc->typecode != SHARED_CONTAINER_TYPE); *typecode = sc->typecode; container_t *answer; @@ -12497,47 +14767,470 @@ void shared_container_free(shared_container_t *container) { } } -extern inline container_t *container_not( - const container_t *c1, uint8_t type1, - uint8_t *result_type); +extern inline container_t *container_not(const container_t *c1, uint8_t type1, + uint8_t *result_type); -extern inline container_t *container_not_range( - const container_t *c1, uint8_t type1, - uint32_t range_start, uint32_t range_end, - uint8_t *result_type); +extern inline container_t *container_not_range(const container_t *c1, + uint8_t type1, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); -extern inline container_t *container_inot( - container_t *c1, uint8_t type1, - uint8_t *result_type); +extern inline container_t *container_inot(container_t *c1, uint8_t type1, + uint8_t *result_type); -extern inline container_t *container_inot_range( - container_t *c1, uint8_t type1, - uint32_t range_start, uint32_t range_end, - uint8_t *result_type); +extern inline container_t *container_inot_range(container_t *c1, uint8_t type1, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); -extern inline container_t *container_range_of_ones( - uint32_t range_start, uint32_t range_end, - uint8_t *result_type); +extern inline container_t *container_range_of_ones(uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); // where are the correponding things for union and intersection?? -extern inline container_t *container_lazy_xor( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +extern inline container_t *container_lazy_xor(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_lazy_ixor(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_andnot(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +roaring_container_iterator_t container_init_iterator(const container_t *c, + uint8_t typecode, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = 0; + uint64_t word; + while ((word = bc->words[wordindex]) == 0) { + wordindex++; + } + // word is non-zero + int32_t index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = index; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + *value = ac->array[0]; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = 0, + }; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + *value = rc->runs[0].value; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = 0, + }; + } + default: + assert(false); + roaring_unreachable; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{0}; + } +} + +roaring_container_iterator_t container_init_iterator_last(const container_t *c, + uint8_t typecode, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = BITSET_CONTAINER_SIZE_IN_WORDS - 1; + uint64_t word; + while ((word = bc->words[wordindex]) == 0) { + wordindex--; + } + // word is non-zero + int32_t index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value = index; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + int32_t index = ac->cardinality - 1; + *value = ac->array[index]; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + int32_t run_index = rc->n_runs - 1; + const rle16_t *last_run = &rc->runs[run_index]; + *value = last_run->value + last_run->length; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = run_index, + }; + } + default: + assert(false); + roaring_unreachable; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{0}; + } +} + +bool container_iterator_next(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + it->index++; + + uint32_t wordindex = it->index / 64; + if (wordindex >= BITSET_CONTAINER_SIZE_IN_WORDS) { + return false; + } + + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + // next part could be optimized/simplified + while (word == 0 && + (wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS)) { + wordindex++; + word = bc->words[wordindex]; + } + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + it->index++; + if (it->index < ac->cardinality) { + *value = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + if (*value == UINT16_MAX) { // Avoid overflow to zero + return false; + } + + const run_container_t *rc = const_CAST_run(c); + uint32_t limit = + rc->runs[it->index].value + rc->runs[it->index].length; + if (*value < limit) { + (*value)++; + return true; + } + + it->index++; + if (it->index < rc->n_runs) { + *value = rc->runs[it->index].value; + return true; + } + return false; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_prev(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + if (--it->index < 0) { + return false; + } + + const bitset_container_t *bc = const_CAST_bitset(c); + int32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + + while (word == 0 && --wordindex >= 0) { + word = bc->words[wordindex]; + } + if (word == 0) { + return false; + } + + it->index = (wordindex * 64) + (63 - roaring_leading_zeroes(word)); + *value = it->index; + return true; + } + case ARRAY_CONTAINER_TYPE: { + if (--it->index < 0) { + return false; + } + const array_container_t *ac = const_CAST_array(c); + *value = ac->array[it->index]; + return true; + } + case RUN_CONTAINER_TYPE: { + if (*value == 0) { + return false; + } + + const run_container_t *rc = const_CAST_run(c); + (*value)--; + if (*value >= rc->runs[it->index].value) { + return true; + } + + if (--it->index < 0) { + return false; + } + + *value = rc->runs[it->index].value + rc->runs[it->index].length; + return true; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value_out, uint16_t val) { + if (val > container_maximum(c, typecode)) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + it->index = bitset_container_index_equalorlarger(bc, val); + *value_out = it->index; + return true; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + it->index = array_container_index_equalorlarger(ac, val); + *value_out = ac->array[it->index]; + return true; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + it->index = run_container_index_equalorlarger(rc, val); + if (rc->runs[it->index].value <= val) { + *value_out = val; + } else { + *value_out = rc->runs[it->index].value; + } + return true; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high16 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | ac->array[it->index + i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} -extern inline container_t *container_lazy_ixor( - container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); +bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high48 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); -extern inline container_t *container_andnot( - const container_t *c1, uint8_t type1, - const container_t *c2, uint8_t type2, - uint8_t *result_type); + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | ac->array[it->index + i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif + +#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T /* end file src/containers/containers.c */ /* begin file src/containers/convert.c */ #include @@ -12546,11 +15239,13 @@ extern inline container_t *container_andnot( #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 #endif #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif // file contains grubby stuff that must know impl. details of all container @@ -12594,25 +15289,25 @@ array_container_t *array_container_from_bitset(const bitset_container_t *bits) { result->cardinality = bits->cardinality; #if CROARING_IS_X64 #if CROARING_COMPILER_SUPPORTS_AVX512 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX512 ) { - bitset_extract_setbits_avx512_uint16(bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, - result->array, bits->cardinality , 0); + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + bitset_extract_setbits_avx512_uint16( + bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, + bits->cardinality, 0); } else #endif { // sse version ends up being slower here // (bitset_extract_setbits_sse_uint16) // because of the sparsity of the data - bitset_extract_setbits_uint16(bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, - result->array, 0); + bitset_extract_setbits_uint16( + bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, 0); } #else - // If the system is not x64, then we have no accelerated function. - bitset_extract_setbits_uint16(bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, + // If the system is not x64, then we have no accelerated function. + bitset_extract_setbits_uint16(bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, 0); #endif - return result; } @@ -12651,10 +15346,9 @@ run_container_t *run_container_from_array(const array_container_t *c) { * Allocates and returns new container, which caller is responsible for freeing. * It does not free the run container. */ -container_t *convert_to_bitset_or_array_container( - run_container_t *rc, int32_t card, - uint8_t *resulttype -){ +container_t *convert_to_bitset_or_array_container(run_container_t *rc, + int32_t card, + uint8_t *resulttype) { if (card <= DEFAULT_MAX_SIZE) { array_container_t *answer = array_container_create_given_capacity(card); answer->cardinality = 0; @@ -12669,7 +15363,7 @@ container_t *convert_to_bitset_or_array_container( } assert(card == answer->cardinality); *resulttype = ARRAY_CONTAINER_TYPE; - //run_container_free(r); + // run_container_free(r); return answer; } bitset_container_t *answer = bitset_container_create(); @@ -12679,7 +15373,7 @@ container_t *convert_to_bitset_or_array_container( } answer->cardinality = card; *resulttype = BITSET_CONTAINER_TYPE; - //run_container_free(r); + // run_container_free(r); return answer; } @@ -12688,10 +15382,8 @@ container_t *convert_to_bitset_or_array_container( /* If a conversion occurs, the caller is responsible to free the original * container and * he becomes responsible to free the new one. */ -container_t *convert_run_to_efficient_container( - run_container_t *c, - uint8_t *typecode_after -){ +container_t *convert_run_to_efficient_container(run_container_t *c, + uint8_t *typecode_after) { int32_t size_as_run_container = run_container_serialized_size_in_bytes(c->n_runs); @@ -12740,9 +15432,7 @@ container_t *convert_run_to_efficient_container( // like convert_run_to_efficient_container but frees the old result if needed container_t *convert_run_to_efficient_container_and_free( - run_container_t *c, - uint8_t *typecode_after -){ + run_container_t *c, uint8_t *typecode_after) { container_t *answer = convert_run_to_efficient_container(c, typecode_after); if (answer != c) run_container_free(c); return answer; @@ -12755,13 +15445,11 @@ container_t *convert_run_to_efficient_container_and_free( // TODO: split into run- array- and bitset- subfunctions for sanity; // a few function calls won't really matter. -container_t *convert_run_optimize( - container_t *c, uint8_t typecode_original, - uint8_t *typecode_after -){ +container_t *convert_run_optimize(container_t *c, uint8_t typecode_original, + uint8_t *typecode_after) { if (typecode_original == RUN_CONTAINER_TYPE) { - container_t *newc = convert_run_to_efficient_container( - CAST_run(c), typecode_after); + container_t *newc = + convert_run_to_efficient_container(CAST_run(c), typecode_after); if (newc != c) { container_free(c, typecode_original); } @@ -12863,10 +15551,8 @@ container_t *convert_run_optimize( } } -container_t *container_from_run_range( - const run_container_t *run, - uint32_t min, uint32_t max, uint8_t *typecode_after -){ +container_t *container_from_run_range(const run_container_t *run, uint32_t min, + uint32_t max, uint8_t *typecode_after) { // We expect most of the time to end up with a bitset container bitset_container_t *bitset = bitset_container_create(); *typecode_after = BITSET_CONTAINER_TYPE; @@ -12878,12 +15564,13 @@ container_t *container_from_run_range( union_cardinality += run->runs[i].length + 1; } union_cardinality += max - min + 1; - union_cardinality -= bitset_lenrange_cardinality(bitset->words, min, max-min); + union_cardinality -= + bitset_lenrange_cardinality(bitset->words, min, max - min); bitset_set_lenrange(bitset->words, min, max - min); bitset->cardinality = union_cardinality; - if(bitset->cardinality <= DEFAULT_MAX_SIZE) { + if (bitset->cardinality <= DEFAULT_MAX_SIZE) { // we need to convert to an array container - array_container_t * array = array_container_from_bitset(bitset); + array_container_t *array = array_container_from_bitset(bitset); *typecode_after = ARRAY_CONTAINER_TYPE; bitset_container_free(bitset); return array; @@ -12892,7 +15579,9 @@ container_t *container_from_run_range( } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/convert.c */ /* begin file src/containers/mixed_andnot.c */ @@ -12906,7 +15595,9 @@ container_t *container_from_run_range( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the andnot of src_1 and src_2 and write the result to @@ -12941,10 +15632,9 @@ void array_bitset_container_iandnot(array_container_t *src_1, * Return true for a bitset result; false for array */ -bool bitset_array_container_andnot( - const bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool bitset_array_container_andnot(const bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { // Java did this directly, but we have option of asm or avx bitset_container_t *result = bitset_container_create(); bitset_container_copy(src_1, result); @@ -12969,10 +15659,9 @@ bool bitset_array_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_array_container_iandnot( - bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool bitset_array_container_iandnot(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { *dst = src_1; src_1->cardinality = (int32_t)bitset_clear_list(src_1->words, (uint64_t)src_1->cardinality, @@ -12993,10 +15682,9 @@ bool bitset_array_container_iandnot( * result true) or an array container. */ -bool run_bitset_container_andnot( - const run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool run_bitset_container_andnot(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { // follows the Java implementation as of June 2016 int card = run_container_cardinality(src_1); if (card <= DEFAULT_MAX_SIZE) { @@ -13049,10 +15737,9 @@ bool run_bitset_container_andnot( * result true) or an array container. */ -bool run_bitset_container_iandnot( - run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool run_bitset_container_iandnot(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { // dummy implementation bool ans = run_bitset_container_andnot(src_1, src_2, dst); run_container_free(src_1); @@ -13066,10 +15753,9 @@ bool run_bitset_container_iandnot( * result true) or an array container. */ -bool bitset_run_container_andnot( - const bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +bool bitset_run_container_andnot(const bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { // follows Java implementation bitset_container_t *result = bitset_container_create(); @@ -13097,10 +15783,9 @@ bool bitset_run_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_run_container_iandnot( - bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +bool bitset_run_container_iandnot(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { *dst = src_1; for (int32_t rlepos = 0; rlepos < src_2->n_runs; ++rlepos) { @@ -13170,10 +15855,9 @@ static int run_array_array_subtract(const run_container_t *rc, * can become any type of container. */ -int run_array_container_andnot( - const run_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +int run_array_container_andnot(const run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { // follows the Java impl as of June 2016 int card = run_container_cardinality(src_1); @@ -13199,7 +15883,7 @@ int run_array_container_andnot( if (end <= xstart) { // output the first run answer->runs[answer->n_runs++] = - MAKE_RLE16(start, end - start - 1); + CROARING_MAKE_RLE16(start, end - start - 1); rlepos++; if (rlepos < src_1->n_runs) { start = src_1->runs[rlepos].value; @@ -13214,7 +15898,7 @@ int run_array_container_andnot( } else { if (start < xstart) { answer->runs[answer->n_runs++] = - MAKE_RLE16(start, xstart - start - 1); + CROARING_MAKE_RLE16(start, xstart - start - 1); } if (xstart + 1 < end) { start = xstart + 1; @@ -13228,7 +15912,8 @@ int run_array_container_andnot( } } if (rlepos < src_1->n_runs) { - answer->runs[answer->n_runs++] = MAKE_RLE16(start, end - start - 1); + answer->runs[answer->n_runs++] = + CROARING_MAKE_RLE16(start, end - start - 1); rlepos++; if (rlepos < src_1->n_runs) { memcpy(answer->runs + answer->n_runs, src_1->runs + rlepos, @@ -13253,8 +15938,7 @@ int run_array_container_andnot( } bitset_container_t *ans = bitset_container_from_run(src_1); bool result_is_bitset = bitset_array_container_iandnot(ans, src_2, dst); - return (result_is_bitset ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE); + return (result_is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE); } /* Compute the andnot of src_1 and src_2 and write the result to @@ -13264,10 +15948,9 @@ int run_array_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -int run_array_container_iandnot( - run_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +int run_array_container_iandnot(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { // dummy implementation same as June 2016 Java int ans = run_array_container_andnot(src_1, src_2, dst); run_container_free(src_1); @@ -13331,10 +16014,8 @@ void array_run_container_iandnot(array_container_t *src_1, * can become any kind of container. */ -int run_run_container_andnot( - const run_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int run_run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { run_container_t *ans = run_container_create(); run_container_andnot(src_1, src_2, ans); uint8_t typecode_after; @@ -13349,10 +16030,8 @@ int run_run_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -int run_run_container_iandnot( - run_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int run_run_container_iandnot(run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { // following Java impl as of June 2016 (dummy) int ans = run_run_container_andnot(src_1, src_2, dst); run_container_free(src_1); @@ -13381,10 +16060,9 @@ void array_array_container_iandnot(array_container_t *src_1, * "dst is a bitset" */ -bool bitset_bitset_container_andnot( - const bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool bitset_bitset_container_andnot(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bitset_container_t *ans = bitset_container_create(); int card = bitset_container_andnot(src_1, src_2, ans); if (card <= DEFAULT_MAX_SIZE) { @@ -13404,10 +16082,9 @@ bool bitset_bitset_container_andnot( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_bitset_container_iandnot( - bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool bitset_bitset_container_iandnot(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { int card = bitset_container_andnot(src_1, src_2, src_1); if (card <= DEFAULT_MAX_SIZE) { *dst = array_container_from_bitset(src_1); @@ -13420,13 +16097,17 @@ bool bitset_bitset_container_iandnot( } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_andnot.c */ /* begin file src/containers/mixed_equal.c */ #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif bool array_container_equal_bitset(const array_container_t* container1, @@ -13479,11 +16160,10 @@ bool run_container_equals_array(const run_container_t* container1, bool run_container_equals_bitset(const run_container_t* container1, const bitset_container_t* container2) { - int run_card = run_container_cardinality(container1); - int bitset_card = (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) ? - container2->cardinality : - bitset_container_compute_cardinality(container2); + int bitset_card = (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) + ? container2->cardinality + : bitset_container_compute_cardinality(container2); if (bitset_card != run_card) { return false; } @@ -13506,7 +16186,9 @@ bool run_container_equals_bitset(const run_container_t* container1, } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_equal.c */ /* begin file src/containers/mixed_intersection.c */ @@ -13517,7 +16199,9 @@ bool run_container_equals_bitset(const run_container_t* container1, #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the intersection of src_1 and src_2 and write the result to @@ -13563,15 +16247,14 @@ int array_bitset_container_intersection_cardinality( return newcard; } - bool array_bitset_container_intersect(const array_container_t *src_1, - const bitset_container_t *src_2) { - const int32_t origcard = src_1->cardinality; - for (int i = 0; i < origcard; ++i) { - uint16_t key = src_1->array[i]; - if(bitset_container_contains(src_2, key)) return true; - } - return false; + const bitset_container_t *src_2) { + const int32_t origcard = src_1->cardinality; + for (int i = 0; i < origcard; ++i) { + uint16_t key = src_1->array[i]; + if (bitset_container_contains(src_2, key)) return true; + } + return false; } /* Compute the intersection of src_1 and src_2 and write the result to @@ -13621,10 +16304,9 @@ void array_run_container_intersection(const array_container_t *src_1, * *dst. If the result is true then the result is a bitset_container_t * otherwise is a array_container_t. If *dst == src_2, an in-place processing * is attempted.*/ -bool run_bitset_container_intersection( - const run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool run_bitset_container_intersection(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { if (run_container_is_full(src_1)) { if (*dst != src_2) *dst = bitset_container_clone(src_2); return true; @@ -13759,13 +16441,12 @@ int run_bitset_container_intersection_cardinality( return answer; } - bool array_run_container_intersect(const array_container_t *src_1, - const run_container_t *src_2) { - if( run_container_is_full(src_2) ) { - return !array_container_empty(src_1); - } - if (src_2->n_runs == 0) { + const run_container_t *src_2) { + if (run_container_is_full(src_2)) { + return !array_container_empty(src_1); + } + if (src_2->n_runs == 0) { return false; } int32_t rlepos = 0; @@ -13794,15 +16475,16 @@ bool array_run_container_intersect(const array_container_t *src_1, /* Compute the intersection between src_1 and src_2 **/ bool run_bitset_container_intersect(const run_container_t *src_1, - const bitset_container_t *src_2) { - if( run_container_is_full(src_1) ) { - return !bitset_container_empty(src_2); - } - for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { - rle16_t rle = src_1->runs[rlepos]; - if(!bitset_lenrange_empty(src_2->words, rle.value,rle.length)) return true; - } - return false; + const bitset_container_t *src_2) { + if (run_container_is_full(src_1)) { + return !bitset_container_empty(src_2); + } + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + if (!bitset_lenrange_empty(src_2->words, rle.value, rle.length)) + return true; + } + return false; } /* @@ -13810,10 +16492,9 @@ bool run_bitset_container_intersect(const run_container_t *src_1, * to *dst. If the return function is true, the result is a bitset_container_t * otherwise is a array_container_t. */ -bool bitset_bitset_container_intersection( - const bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool bitset_bitset_container_intersection(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { const int newCardinality = bitset_container_and_justcard(src_1, src_2); if (newCardinality > DEFAULT_MAX_SIZE) { *dst = bitset_container_create(); @@ -13835,8 +16516,7 @@ bool bitset_bitset_container_intersection( bool bitset_bitset_container_intersection_inplace( bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ + container_t **dst) { const int newCardinality = bitset_container_and_justcard(src_1, src_2); if (newCardinality > DEFAULT_MAX_SIZE) { *dst = src_1; @@ -13855,7 +16535,9 @@ bool bitset_bitset_container_intersection_inplace( } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_intersection.c */ /* begin file src/containers/mixed_negation.c */ @@ -13869,7 +16551,9 @@ bool bitset_bitset_container_intersection_inplace( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif // TODO: make simplified and optimized negation code across @@ -13902,9 +16586,8 @@ void array_container_negation(const array_container_t *src, * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -bool bitset_container_negation( - const bitset_container_t *src, container_t **dst -){ +bool bitset_container_negation(const bitset_container_t *src, + container_t **dst) { return bitset_container_negation_range(src, 0, (1 << 16), dst); } @@ -13917,9 +16600,8 @@ bool bitset_container_negation( * to free the container. * In all cases, the result is in *dst. */ -bool bitset_container_negation_inplace( - bitset_container_t *src, container_t **dst -){ +bool bitset_container_negation_inplace(bitset_container_t *src, + container_t **dst) { return bitset_container_negation_range_inplace(src, 0, (1 << 16), dst); } @@ -13949,11 +16631,9 @@ int run_container_negation_inplace(run_container_t *src, container_t **dst) { * to *dst. Returns true if the result is a bitset container * and false for an array container. *dst is not preallocated. */ -bool array_container_negation_range( - const array_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +bool array_container_negation_range(const array_container_t *src, + const int range_start, const int range_end, + container_t **dst) { /* close port of the Java implementation */ if (range_start >= range_end) { *dst = array_container_clone(src); @@ -13988,9 +16668,9 @@ bool array_container_negation_range( array_container_t *arr = array_container_create_given_capacity(new_cardinality); *dst = (container_t *)arr; - if(new_cardinality == 0) { - arr->cardinality = new_cardinality; - return false; // we are done. + if (new_cardinality == 0) { + arr->cardinality = new_cardinality; + return false; // we are done. } // copy stuff before the active area memcpy(arr->array, src->array, start_index * sizeof(uint16_t)); @@ -14019,11 +16699,10 @@ bool array_container_negation_range( * inplace version without inefficient copying. */ -bool array_container_negation_range_inplace( - array_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +bool array_container_negation_range_inplace(array_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { bool ans = array_container_negation_range(src, range_start, range_end, dst); // TODO : try a real inplace version array_container_free(src); @@ -14037,11 +16716,9 @@ bool array_container_negation_range_inplace( * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -bool bitset_container_negation_range( - const bitset_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +bool bitset_container_negation_range(const bitset_container_t *src, + const int range_start, const int range_end, + container_t **dst) { // TODO maybe consider density-based estimate // and sometimes build result directly as array, with // conversion back to bitset if wrong. Or determine @@ -14071,11 +16748,10 @@ bool bitset_container_negation_range( * to free the container. * In all cases, the result is in *dst. */ -bool bitset_container_negation_range_inplace( - bitset_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +bool bitset_container_negation_range_inplace(bitset_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { bitset_flip_range(src->words, (uint32_t)range_start, (uint32_t)range_end); src->cardinality = bitset_container_compute_cardinality(src); if (src->cardinality > DEFAULT_MAX_SIZE) { @@ -14093,11 +16769,9 @@ bool bitset_container_negation_range_inplace( * We assume that dst is not pre-allocated. In * case of failure, *dst will be NULL. */ -int run_container_negation_range( - const run_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +int run_container_negation_range(const run_container_t *src, + const int range_start, const int range_end, + container_t **dst) { uint8_t return_typecode; // follows the Java implementation @@ -14135,11 +16809,10 @@ int run_container_negation_range( * then src is modified and no allocation is made. * In all cases, the result is in *dst. */ -int run_container_negation_range_inplace( - run_container_t *src, - const int range_start, const int range_end, - container_t **dst -){ +int run_container_negation_range_inplace(run_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { uint8_t return_typecode; if (range_end <= range_start) { @@ -14190,7 +16863,7 @@ int run_container_negation_range_inplace( } // as with Java implementation, use locals to give self a buffer of depth 1 - rle16_t buffered = MAKE_RLE16(0, 0); + rle16_t buffered = CROARING_MAKE_RLE16(0, 0); rle16_t next = buffered; if (k < my_nbr_runs) buffered = src->runs[k]; @@ -14212,13 +16885,17 @@ int run_container_negation_range_inplace( } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_negation.c */ /* begin file src/containers/mixed_subset.c */ #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif bool array_container_is_subset_bitset(const array_container_t* container1, @@ -14353,7 +17030,9 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_subset.c */ /* begin file src/containers/mixed_union.c */ @@ -14367,7 +17046,9 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the union of src_1 and src_2 and write the result to @@ -14512,17 +17193,16 @@ void array_run_container_inplace_union(const array_container_t *src_1, } } -bool array_array_container_union( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; if (totalCardinality <= DEFAULT_MAX_SIZE) { *dst = array_container_create_given_capacity(totalCardinality); if (*dst != NULL) { array_container_union(src_1, src_2, CAST_array(*dst)); } else { - return true; // otherwise failure won't be caught + return true; // otherwise failure won't be caught } return false; // not a bitset } @@ -14544,30 +17224,32 @@ bool array_array_container_union( return returnval; } -bool array_array_container_inplace_union( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; *dst = NULL; if (totalCardinality <= DEFAULT_MAX_SIZE) { - if(src_1->capacity < totalCardinality) { - *dst = array_container_create_given_capacity(2 * totalCardinality); // be purposefully generous - if (*dst != NULL) { - array_container_union(src_1, src_2, CAST_array(*dst)); - } else { - return true; // otherwise failure won't be caught - } - return false; // not a bitset + if (src_1->capacity < totalCardinality) { + *dst = array_container_create_given_capacity( + 2 * totalCardinality); // be purposefully generous + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset } else { - memmove(src_1->array + src_2->cardinality, src_1->array, src_1->cardinality * sizeof(uint16_t)); - // In theory, we could use fast_union_uint16, but it is unsafe. It fails - // with Intel compilers in particular. - // https://github.com/RoaringBitmap/CRoaring/pull/452 - // See report https://github.com/RoaringBitmap/CRoaring/issues/476 - src_1->cardinality = (int32_t)union_uint16(src_1->array + src_2->cardinality, src_1->cardinality, - src_2->array, src_2->cardinality, src_1->array); - return false; // not a bitset + memmove(src_1->array + src_2->cardinality, src_1->array, + src_1->cardinality * sizeof(uint16_t)); + // In theory, we could use fast_union_uint16, but it is unsafe. It + // fails with Intel compilers in particular. + // https://github.com/RoaringBitmap/CRoaring/pull/452 + // See report https://github.com/RoaringBitmap/CRoaring/issues/476 + src_1->cardinality = (int32_t)union_uint16( + src_1->array + src_2->cardinality, src_1->cardinality, + src_2->array, src_2->cardinality, src_1->array); + return false; // not a bitset } } *dst = bitset_container_create(); @@ -14580,13 +17262,14 @@ bool array_array_container_inplace_union( src_2->cardinality); if (ourbitset->cardinality <= DEFAULT_MAX_SIZE) { // need to convert! - if(src_1->capacity < ourbitset->cardinality) { - array_container_grow(src_1, ourbitset->cardinality, false); + if (src_1->capacity < ourbitset->cardinality) { + array_container_grow(src_1, ourbitset->cardinality, false); } - bitset_extract_setbits_uint16(ourbitset->words, BITSET_CONTAINER_SIZE_IN_WORDS, - src_1->array, 0); - src_1->cardinality = ourbitset->cardinality; + bitset_extract_setbits_uint16(ourbitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS, + src_1->array, 0); + src_1->cardinality = ourbitset->cardinality; *dst = src_1; bitset_container_free(ourbitset); returnval = false; // not going to be a bitset @@ -14595,29 +17278,28 @@ bool array_array_container_inplace_union( return returnval; } - -bool array_array_container_lazy_union( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_lazy_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; // // We assume that operations involving bitset containers will be faster than - // operations involving solely array containers, except maybe when array containers - // are small. Indeed, for example, it is cheap to compute the union between an array and - // a bitset container, generally more so than between a large array and another array. - // So it is advantageous to favour bitset containers during the computation. - // Of course, if we convert array containers eagerly to bitset containers, we may later - // need to revert the bitset containers to array containerr to satisfy the Roaring format requirements, - // but such one-time conversions at the end may not be overly expensive. We arrived to this design - // based on extensive benchmarking. + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // union between an array and a bitset container, generally more so than + // between a large array and another array. So it is advantageous to favour + // bitset containers during the computation. Of course, if we convert array + // containers eagerly to bitset containers, we may later need to revert the + // bitset containers to array containerr to satisfy the Roaring format + // requirements, but such one-time conversions at the end may not be overly + // expensive. We arrived to this design based on extensive benchmarking. // if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { *dst = array_container_create_given_capacity(totalCardinality); if (*dst != NULL) { array_container_union(src_1, src_2, CAST_array(*dst)); } else { - return true; // otherwise failure won't be caught + return true; // otherwise failure won't be caught } return false; // not a bitset } @@ -14632,68 +17314,78 @@ bool array_array_container_lazy_union( return returnval; } - -bool array_array_container_lazy_inplace_union( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_lazy_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; *dst = NULL; // // We assume that operations involving bitset containers will be faster than - // operations involving solely array containers, except maybe when array containers - // are small. Indeed, for example, it is cheap to compute the union between an array and - // a bitset container, generally more so than between a large array and another array. - // So it is advantageous to favour bitset containers during the computation. - // Of course, if we convert array containers eagerly to bitset containers, we may later - // need to revert the bitset containers to array containerr to satisfy the Roaring format requirements, - // but such one-time conversions at the end may not be overly expensive. We arrived to this design - // based on extensive benchmarking. + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // union between an array and a bitset container, generally more so than + // between a large array and another array. So it is advantageous to favour + // bitset containers during the computation. Of course, if we convert array + // containers eagerly to bitset containers, we may later need to revert the + // bitset containers to array containerr to satisfy the Roaring format + // requirements, but such one-time conversions at the end may not be overly + // expensive. We arrived to this design based on extensive benchmarking. // if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { - if(src_1->capacity < totalCardinality) { - *dst = array_container_create_given_capacity(2 * totalCardinality); // be purposefully generous - if (*dst != NULL) { - array_container_union(src_1, src_2, CAST_array(*dst)); - } else { - return true; // otherwise failure won't be caught - } - return false; // not a bitset + if (src_1->capacity < totalCardinality) { + *dst = array_container_create_given_capacity( + 2 * totalCardinality); // be purposefully generous + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset } else { - memmove(src_1->array + src_2->cardinality, src_1->array, src_1->cardinality * sizeof(uint16_t)); - /* - Next line is safe: - - We just need to focus on the reading and writing performed on array1. In `union_vector16`, both vectorized and scalar code still obey the basic rule: read from two inputs, do the union, and then write the output. - - Let's say the length(cardinality) of input2 is L2: - ``` - |<- L2 ->| - array1: [output--- |input 1---|---] - array2: [input 2---] - ``` - Let's define 3 __m128i pointers, `pos1` starts from `input1`, `pos2` starts from `input2`, these 2 point at the next byte to read, `out` starts from `output`, pointing at the next byte to overwrite. - ``` - array1: [output--- |input 1---|---] - ^ ^ - out pos1 - array2: [input 2---] - ^ - pos2 - ``` - The union output always contains less or equal number of elements than all inputs added, so we have: - ``` - out <= pos1 + pos2 - ``` - therefore: - ``` - out <= pos1 + L2 - ``` - which means you will not overwrite data beyond pos1, so the data haven't read is safe, and we don't care the data already read. - */ - src_1->cardinality = (int32_t)fast_union_uint16(src_1->array + src_2->cardinality, src_1->cardinality, - src_2->array, src_2->cardinality, src_1->array); - return false; // not a bitset + memmove(src_1->array + src_2->cardinality, src_1->array, + src_1->cardinality * sizeof(uint16_t)); + /* + Next line is safe: + + We just need to focus on the reading and writing performed on + array1. In `union_vector16`, both vectorized and scalar code still + obey the basic rule: read from two inputs, do the union, and then + write the output. + + Let's say the length(cardinality) of input2 is L2: + ``` + |<- L2 ->| + array1: [output--- |input 1---|---] + array2: [input 2---] + ``` + Let's define 3 __m128i pointers, `pos1` starts from `input1`, + `pos2` starts from `input2`, these 2 point at the next byte to + read, `out` starts from `output`, pointing at the next byte to + overwrite. + ``` + array1: [output--- |input 1---|---] + ^ ^ + out pos1 + array2: [input 2---] + ^ + pos2 + ``` + The union output always contains less or equal number of elements + than all inputs added, so we have: + ``` + out <= pos1 + pos2 + ``` + therefore: + ``` + out <= pos1 + L2 + ``` + which means you will not overwrite data beyond pos1, so the data + haven't read is safe, and we don't care the data already read. + */ + src_1->cardinality = (int32_t)fast_union_uint16( + src_1->array + src_2->cardinality, src_1->cardinality, + src_2->array, src_2->cardinality, src_1->array); + return false; // not a bitset } } *dst = bitset_container_create(); @@ -14708,7 +17400,9 @@ bool array_array_container_lazy_inplace_union( } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_union.c */ /* begin file src/containers/mixed_xor.c */ @@ -14721,16 +17415,17 @@ bool array_array_container_lazy_inplace_union( #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif /* Compute the xor of src_1 and src_2 and write the result to * dst (which has no container initially). * Result is true iff dst is a bitset */ -bool array_bitset_container_xor( - const array_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool array_bitset_container_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bitset_container_t *result = bitset_container_create(); bitset_container_copy(src_2, result); result->cardinality = (int32_t)bitset_flip_list_withcard( @@ -14766,10 +17461,9 @@ void array_bitset_container_lazy_xor(const array_container_t *src_1, * result true) or an array container. */ -bool run_bitset_container_xor( - const run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool run_bitset_container_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bitset_container_t *result = bitset_container_create(); bitset_container_copy(src_2, result); @@ -14810,10 +17504,8 @@ void run_bitset_container_lazy_xor(const run_container_t *src_1, * can become any kind of container. */ -int array_run_container_xor( - const array_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int array_run_container_xor(const array_container_t *src_1, + const run_container_t *src_2, container_t **dst) { // semi following Java XOR implementation as of May 2016 // the C OR implementation works quite differently and can return a run // container @@ -14837,15 +17529,13 @@ int array_run_container_xor( array_container_t *temp = array_container_from_run(src_2); bool ret_is_bitset = array_array_container_xor(temp, src_1, dst); array_container_free(temp); - return ret_is_bitset ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE; + return ret_is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE; } else { // guess that it will end up as a bitset bitset_container_t *result = bitset_container_from_run(src_2); bool is_bitset = bitset_array_container_ixor(result, src_1, dst); // any necessary type conversion has been done by the ixor - int retval = (is_bitset ? BITSET_CONTAINER_TYPE - : ARRAY_CONTAINER_TYPE); + int retval = (is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE); return retval; } } @@ -14889,10 +17579,8 @@ void array_run_container_lazy_xor(const array_container_t *src_1, * can become any kind of container. */ -int run_run_container_xor( - const run_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int run_run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { run_container_t *ans = run_container_create(); run_container_xor(src_1, src_2, ans); uint8_t typecode_after; @@ -14908,10 +17596,9 @@ int run_run_container_xor( * */ -bool array_array_container_xor( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; // upper bound if (totalCardinality <= DEFAULT_MAX_SIZE) { @@ -14934,28 +17621,28 @@ bool array_array_container_xor( return returnval; } -bool array_array_container_lazy_xor( - const array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_lazy_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int totalCardinality = src_1->cardinality + src_2->cardinality; // // We assume that operations involving bitset containers will be faster than - // operations involving solely array containers, except maybe when array containers - // are small. Indeed, for example, it is cheap to compute the exclusive union between an array and - // a bitset container, generally more so than between a large array and another array. - // So it is advantageous to favour bitset containers during the computation. - // Of course, if we convert array containers eagerly to bitset containers, we may later - // need to revert the bitset containers to array containerr to satisfy the Roaring format requirements, - // but such one-time conversions at the end may not be overly expensive. We arrived to this design - // based on extensive benchmarking on unions. - // For XOR/exclusive union, we simply followed the heuristic used by the unions (see mixed_union.c). - // Further tuning is possible. + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // exclusive union between an array and a bitset container, generally more + // so than between a large array and another array. So it is advantageous to + // favour bitset containers during the computation. Of course, if we convert + // array containers eagerly to bitset containers, we may later need to + // revert the bitset containers to array containerr to satisfy the Roaring + // format requirements, but such one-time conversions at the end may not be + // overly expensive. We arrived to this design based on extensive + // benchmarking on unions. For XOR/exclusive union, we simply followed the + // heuristic used by the unions (see mixed_union.c). Further tuning is + // possible. // if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { *dst = array_container_create_given_capacity(totalCardinality); - if (*dst != NULL) - array_container_xor(src_1, src_2, CAST_array(*dst)); + if (*dst != NULL) array_container_xor(src_1, src_2, CAST_array(*dst)); return false; // not a bitset } *dst = bitset_container_from_array(src_1); @@ -14973,10 +17660,9 @@ bool array_array_container_lazy_xor( * "dst is a bitset" */ -bool bitset_bitset_container_xor( - const bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool bitset_bitset_container_xor(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bitset_container_t *ans = bitset_container_create(); int card = bitset_container_xor(src_1, src_2, ans); if (card <= DEFAULT_MAX_SIZE) { @@ -14996,10 +17682,9 @@ bool bitset_bitset_container_xor( * cases, the caller is responsible for deallocating dst. * Returns true iff dst is a bitset */ -bool bitset_array_container_ixor( - bitset_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool bitset_array_container_ixor(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { *dst = src_1; src_1->cardinality = (uint32_t)bitset_flip_list_withcard( src_1->words, src_1->cardinality, src_2->array, src_2->cardinality); @@ -15017,10 +17702,9 @@ bool bitset_array_container_ixor( * Anything inplace with a bitset is a good candidate */ -bool bitset_bitset_container_ixor( - bitset_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool bitset_bitset_container_ixor(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { int card = bitset_container_xor(src_1, src_2, src_1); if (card <= DEFAULT_MAX_SIZE) { *dst = array_container_from_bitset(src_1); @@ -15032,10 +17716,9 @@ bool bitset_bitset_container_ixor( } } -bool array_bitset_container_ixor( - array_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool array_bitset_container_ixor(array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bool ans = array_bitset_container_xor(src_1, src_2, dst); array_container_free(src_1); return ans; @@ -15048,19 +17731,17 @@ bool array_bitset_container_ixor( * result true) or an array container. */ -bool run_bitset_container_ixor( - run_container_t *src_1, const bitset_container_t *src_2, - container_t **dst -){ +bool run_bitset_container_ixor(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { bool ans = run_bitset_container_xor(src_1, src_2, dst); run_container_free(src_1); return ans; } -bool bitset_run_container_ixor( - bitset_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +bool bitset_run_container_ixor(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { bool ans = run_bitset_container_xor(src_2, src_1, dst); bitset_container_free(src_1); return ans; @@ -15070,44 +17751,40 @@ bool bitset_run_container_ixor( * can become any kind of container. */ -int array_run_container_ixor( - array_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int array_run_container_ixor(array_container_t *src_1, + const run_container_t *src_2, container_t **dst) { int ans = array_run_container_xor(src_1, src_2, dst); array_container_free(src_1); return ans; } -int run_array_container_ixor( - run_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +int run_array_container_ixor(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { int ans = array_run_container_xor(src_2, src_1, dst); run_container_free(src_1); return ans; } -bool array_array_container_ixor( - array_container_t *src_1, const array_container_t *src_2, - container_t **dst -){ +bool array_array_container_ixor(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { bool ans = array_array_container_xor(src_1, src_2, dst); array_container_free(src_1); return ans; } -int run_run_container_ixor( - run_container_t *src_1, const run_container_t *src_2, - container_t **dst -){ +int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, + container_t **dst) { int ans = run_run_container_xor(src_1, src_2, dst); run_container_free(src_1); return ans; } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/containers/mixed_xor.c */ /* begin file src/containers/run.c */ @@ -15118,11 +17795,17 @@ int run_run_container_ixor( #if CROARING_IS_X64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif - #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif extern inline uint16_t run_container_minimum(const run_container_t *run); @@ -15131,15 +17814,15 @@ extern inline int32_t interleavedBinarySearch(const rle16_t *array, int32_t lenarray, uint16_t ikey); extern inline bool run_container_contains(const run_container_t *run, uint16_t pos); -extern inline int run_container_index_equalorlarger(const run_container_t *arr, uint16_t x); +extern inline int run_container_index_equalorlarger(const run_container_t *arr, + uint16_t x); extern inline bool run_container_is_full(const run_container_t *run); extern inline bool run_container_nonzero_cardinality(const run_container_t *rc); extern inline int32_t run_container_serialized_size_in_bytes(int32_t num_runs); extern inline run_container_t *run_container_create_range(uint32_t start, - uint32_t stop); + uint32_t stop); extern inline int run_container_cardinality(const run_container_t *run); - bool run_container_add(run_container_t *run, uint16_t pos) { int32_t index = interleavedBinarySearch(run->runs, run->n_runs, pos); if (index >= 0) return false; // already there @@ -15193,12 +17876,14 @@ bool run_container_add(run_container_t *run, uint16_t pos) { run_container_t *run_container_create_given_capacity(int32_t size) { run_container_t *run; /* Allocate the run container itself. */ - if ((run = (run_container_t *)roaring_malloc(sizeof(run_container_t))) == NULL) { + if ((run = (run_container_t *)roaring_malloc(sizeof(run_container_t))) == + NULL) { return NULL; } - if (size <= 0 ) { // we don't want to rely on malloc(0) + if (size <= 0) { // we don't want to rely on malloc(0) run->runs = NULL; - } else if ((run->runs = (rle16_t *)roaring_malloc(sizeof(rle16_t) * size)) == NULL) { + } else if ((run->runs = (rle16_t *)roaring_malloc(sizeof(rle16_t) * + size)) == NULL) { roaring_free(run); return NULL; } @@ -15212,7 +17897,8 @@ int run_container_shrink_to_fit(run_container_t *src) { int savings = src->capacity - src->n_runs; src->capacity = src->n_runs; rle16_t *oldruns = src->runs; - src->runs = (rle16_t *)roaring_realloc(oldruns, src->capacity * sizeof(rle16_t)); + src->runs = + (rle16_t *)roaring_realloc(oldruns, src->capacity * sizeof(rle16_t)); if (src->runs == NULL) roaring_free(oldruns); // should never happen? return savings; } @@ -15221,6 +17907,7 @@ run_container_t *run_container_create(void) { return run_container_create_given_capacity(RUN_DEFAULT_INIT_SIZE); } +ALLOW_UNALIGNED run_container_t *run_container_clone(const run_container_t *src) { run_container_t *run = run_container_create_given_capacity(src->capacity); if (run == NULL) return NULL; @@ -15230,9 +17917,8 @@ run_container_t *run_container_clone(const run_container_t *src) { return run; } -void run_container_offset(const run_container_t *c, - container_t **loc, container_t **hic, - uint16_t offset) { +void run_container_offset(const run_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { run_container_t *lo = NULL, *hi = NULL; bool split; @@ -15254,29 +17940,30 @@ void run_container_offset(const run_container_t *c, if (loc && lo_cap) { lo = run_container_create_given_capacity(lo_cap); - memcpy(lo->runs, c->runs, lo_cap*sizeof(rle16_t)); + memcpy(lo->runs, c->runs, lo_cap * sizeof(rle16_t)); lo->n_runs = lo_cap; for (int i = 0; i < lo_cap; ++i) { lo->runs[i].value += offset; } - *loc = (container_t*)lo; + *loc = (container_t *)lo; } if (hic && hi_cap) { hi = run_container_create_given_capacity(hi_cap); - memcpy(hi->runs, c->runs+pivot, hi_cap*sizeof(rle16_t)); + memcpy(hi->runs, c->runs + pivot, hi_cap * sizeof(rle16_t)); hi->n_runs = hi_cap; for (int i = 0; i < hi_cap; ++i) { hi->runs[i].value += offset; } - *hic = (container_t*)hi; + *hic = (container_t *)hi; } // Fix the split. if (split) { if (lo != NULL) { // Add the missing run to 'lo', exhausting length. - lo->runs[lo->n_runs-1].length = (1 << 16) - lo->runs[lo->n_runs-1].value - 1; + lo->runs[lo->n_runs - 1].length = + (1 << 16) - lo->runs[lo->n_runs - 1].value - 1; } if (hi != NULL) { @@ -15289,32 +17976,31 @@ void run_container_offset(const run_container_t *c, /* Free memory. */ void run_container_free(run_container_t *run) { - if(run->runs != NULL) {// Jon Strabala reports that some tools complain otherwise - roaring_free(run->runs); - run->runs = NULL; // pedantic + if (run->runs != + NULL) { // Jon Strabala reports that some tools complain otherwise + roaring_free(run->runs); + run->runs = NULL; // pedantic } roaring_free(run); } void run_container_grow(run_container_t *run, int32_t min, bool copy) { - int32_t newCapacity = - (run->capacity == 0) - ? RUN_DEFAULT_INIT_SIZE - : run->capacity < 64 ? run->capacity * 2 - : run->capacity < 1024 ? run->capacity * 3 / 2 - : run->capacity * 5 / 4; + int32_t newCapacity = (run->capacity == 0) ? RUN_DEFAULT_INIT_SIZE + : run->capacity < 64 ? run->capacity * 2 + : run->capacity < 1024 ? run->capacity * 3 / 2 + : run->capacity * 5 / 4; if (newCapacity < min) newCapacity = min; run->capacity = newCapacity; assert(run->capacity >= min); if (copy) { rle16_t *oldruns = run->runs; - run->runs = - (rle16_t *)roaring_realloc(oldruns, run->capacity * sizeof(rle16_t)); + run->runs = (rle16_t *)roaring_realloc(oldruns, + run->capacity * sizeof(rle16_t)); if (run->runs == NULL) roaring_free(oldruns); } else { // Jon Strabala reports that some tools complain otherwise if (run->runs != NULL) { - roaring_free(run->runs); + roaring_free(run->runs); } run->runs = (rle16_t *)roaring_malloc(run->capacity * sizeof(rle16_t)); } @@ -15638,7 +18324,7 @@ int run_container_intersection_cardinality(const run_container_t *src_1, } bool run_container_intersect(const run_container_t *src_1, - const run_container_t *src_2) { + const run_container_t *src_2) { const bool if1 = run_container_is_full(src_1); const bool if2 = run_container_is_full(src_2); if (if1 || if2) { @@ -15646,7 +18332,7 @@ bool run_container_intersect(const run_container_t *src_1, return !run_container_empty(src_2); } if (if2) { - return !run_container_empty(src_1); + return !run_container_empty(src_1); } } int32_t rlepos = 0; @@ -15675,7 +18361,6 @@ bool run_container_intersect(const run_container_t *src_1, return false; } - /* Compute the difference of src_1 and src_2 and write the result to * dst. It is assumed that dst is distinct from both src_1 and src_2. */ void run_container_andnot(const run_container_t *src_1, @@ -15697,7 +18382,8 @@ void run_container_andnot(const run_container_t *src_1, while ((rlepos1 < src_1->n_runs) && (rlepos2 < src_2->n_runs)) { if (end <= start2) { // output the first run - dst->runs[dst->n_runs++] = MAKE_RLE16(start, end - start - 1); + dst->runs[dst->n_runs++] = + CROARING_MAKE_RLE16(start, end - start - 1); rlepos1++; if (rlepos1 < src_1->n_runs) { start = src_1->runs[rlepos1].value; @@ -15713,7 +18399,7 @@ void run_container_andnot(const run_container_t *src_1, } else { if (start < start2) { dst->runs[dst->n_runs++] = - MAKE_RLE16(start, start2 - start - 1); + CROARING_MAKE_RLE16(start, start2 - start - 1); } if (end2 < end) { start = end2; @@ -15727,7 +18413,7 @@ void run_container_andnot(const run_container_t *src_1, } } if (rlepos1 < src_1->n_runs) { - dst->runs[dst->n_runs++] = MAKE_RLE16(start, end - start - 1); + dst->runs[dst->n_runs++] = CROARING_MAKE_RLE16(start, end - start - 1); rlepos1++; if (rlepos1 < src_1->n_runs) { memcpy(dst->runs + dst->n_runs, src_1->runs + rlepos1, @@ -15804,7 +18490,8 @@ bool run_container_validate(const run_container_t *run, const char **reason) { } if (run->n_runs == 0) { - return true; + *reason = "zero run count"; + return false; } if (run->runs == NULL) { *reason = "NULL runs"; @@ -15820,7 +18507,7 @@ bool run_container_validate(const run_container_t *run, const char **reason) { *reason = "run start + length overflow"; return false; } - if (end > (1<<16)) { + if (end > (1 << 16)) { *reason = "run start + length too large"; return false; } @@ -15853,9 +18540,9 @@ int32_t run_container_read(int32_t cardinality, run_container_t *container, container->n_runs = cast_16; if (container->n_runs > container->capacity) run_container_grow(container, container->n_runs, false); - if(container->n_runs > 0) { - memcpy(container->runs, buf + sizeof(uint16_t), - container->n_runs * sizeof(rle16_t)); + if (container->n_runs > 0) { + memcpy(container->runs, buf + sizeof(uint16_t), + container->n_runs * sizeof(rle16_t)); } return run_container_size_in_bytes(container); } @@ -15928,7 +18615,7 @@ void run_container_smart_append_exclusive(run_container_t *src, if (!src->n_runs || (start > (old_end = last_run->value + last_run->length + 1))) { - *appended_last_run = MAKE_RLE16(start, length); + *appended_last_run = CROARING_MAKE_RLE16(start, length); src->n_runs++; return; } @@ -15942,10 +18629,10 @@ void run_container_smart_append_exclusive(run_container_t *src, if (start == last_run->value) { // wipe out previous if (new_end < old_end) { - *last_run = MAKE_RLE16(new_end, old_end - new_end - 1); + *last_run = CROARING_MAKE_RLE16(new_end, old_end - new_end - 1); return; } else if (new_end > old_end) { - *last_run = MAKE_RLE16(old_end, new_end - old_end - 1); + *last_run = CROARING_MAKE_RLE16(old_end, new_end - old_end - 1); return; } else { src->n_runs--; @@ -15954,10 +18641,12 @@ void run_container_smart_append_exclusive(run_container_t *src, } last_run->length = start - last_run->value - 1; if (new_end < old_end) { - *appended_last_run = MAKE_RLE16(new_end, old_end - new_end - 1); + *appended_last_run = + CROARING_MAKE_RLE16(new_end, old_end - new_end - 1); src->n_runs++; } else if (new_end > old_end) { - *appended_last_run = MAKE_RLE16(old_end, new_end - old_end - 1); + *appended_last_run = + CROARING_MAKE_RLE16(old_end, new_end - old_end - 1); src->n_runs++; } } @@ -15993,6 +18682,40 @@ int run_container_rank(const run_container_t *container, uint16_t x) { } return sum; } +uint32_t run_container_rank_many(const run_container_t *container, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + const uint16_t high = (uint16_t)((*begin) >> 16); + const uint32_t *iter = begin; + int sum = 0; + int i = 0; + for (; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if (xhigh != high) return iter - begin; // stop at next container + + uint32_t x32 = x & 0xFFFF; + while (i < container->n_runs) { + uint32_t startpoint = container->runs[i].value; + uint32_t length = container->runs[i].length; + uint32_t endpoint = length + startpoint; + if (x32 <= endpoint) { + if (x32 < startpoint) { + *(ans++) = start_rank + sum; + } else { + *(ans++) = start_rank + sum + (x32 - startpoint) + 1; + } + break; + } else { + sum += length + 1; + i++; + } + } + if (i >= container->n_runs) *(ans++) = start_rank + sum; + } + + return iter - begin; +} int run_container_get_index(const run_container_t *container, uint16_t x) { if (run_container_contains(container, x)) { @@ -16020,7 +18743,8 @@ int run_container_get_index(const run_container_t *container, uint16_t x) { CROARING_TARGET_AVX512 ALLOW_UNALIGNED /* Get the cardinality of `run'. Requires an actual computation. */ -static inline int _avx512_run_container_cardinality(const run_container_t *run) { +static inline int _avx512_run_container_cardinality( + const run_container_t *run) { const int32_t n_runs = run->n_runs; const rle16_t *runs = run->runs; @@ -16048,7 +18772,6 @@ static inline int _avx512_run_container_cardinality(const run_container_t *run) _mm256_storeu_si256((__m256i *)buffer, hi); sum += (buffer[0] + buffer[1]) + (buffer[2] + buffer[3]) + (buffer[4] + buffer[5]) + (buffer[6] + buffer[7]); - } for (; k < n_runs; ++k) { sum += runs[k].length; @@ -16093,7 +18816,8 @@ static inline int _avx2_run_container_cardinality(const run_container_t *run) { CROARING_UNTARGET_AVX2 /* Get the cardinality of `run'. Requires an actual computation. */ -static inline int _scalar_run_container_cardinality(const run_container_t *run) { +static inline int _scalar_run_container_cardinality( + const run_container_t *run) { const int32_t n_runs = run->n_runs; const rle16_t *runs = run->runs; @@ -16108,20 +18832,20 @@ static inline int _scalar_run_container_cardinality(const run_container_t *run) int run_container_cardinality(const run_container_t *run) { #if CROARING_COMPILER_SUPPORTS_AVX512 - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX512 ) { - return _avx512_run_container_cardinality(run); - } - else + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + return _avx512_run_container_cardinality(run); + } else #endif - if( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { - return _avx2_run_container_cardinality(run); - } else { - return _scalar_run_container_cardinality(run); - } + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _avx2_run_container_cardinality(run); + } else { + return _scalar_run_container_cardinality(run); + } } #else /* Get the cardinality of `run'. Requires an actual computation. */ +ALLOW_UNALIGNED int run_container_cardinality(const run_container_t *run) { const int32_t n_runs = run->n_runs; const rle16_t *runs = run->runs; @@ -16136,11 +18860,14 @@ int run_container_cardinality(const run_container_t *run) { } #endif - #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif -/* end file src/containers/run.c */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif/* end file src/containers/run.c */ /* begin file src/isadetection.c */ /* From @@ -16188,3881 +18915,5846 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -#include -#include +#include +#include +#include + +// Binaries produced by Visual Studio with solely AVX2 routines +// can compile to AVX-512 thus causing crashes on non-AVX-512 systems. +// This appears to affect VS 17.8 and 17.9. We disable AVX-512 and AVX2 +// on these systems. It seems that ClangCL is not affected. +// https://github.com/RoaringBitmap/CRoaring/pull/603 +#ifndef __clang__ +#if _MSC_VER >= 1938 +#define ROARING_DISABLE_AVX 1 +#endif // _MSC_VER >= 1938 +#endif // __clang__ + +// We need portability.h to be included first, see +// https://github.com/RoaringBitmap/CRoaring/issues/394 +#if CROARING_REGULAR_VISUAL_STUDIO +#include +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include +#endif // CROARING_REGULAR_VISUAL_STUDIO + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif +enum croaring_instruction_set { + CROARING_DEFAULT = 0x0, + CROARING_NEON = 0x1, + CROARING_AVX2 = 0x4, + CROARING_SSE42 = 0x8, + CROARING_PCLMULQDQ = 0x10, + CROARING_BMI1 = 0x20, + CROARING_BMI2 = 0x40, + CROARING_ALTIVEC = 0x80, + CROARING_AVX512F = 0x100, + CROARING_AVX512DQ = 0x200, + CROARING_AVX512BW = 0x400, + CROARING_AVX512VBMI2 = 0x800, + CROARING_AVX512BITALG = 0x1000, + CROARING_AVX512VPOPCNTDQ = 0x2000, + CROARING_UNINITIALIZED = 0x8000 +}; + +#if CROARING_COMPILER_SUPPORTS_AVX512 +unsigned int CROARING_AVX512_REQUIRED = + (CROARING_AVX512F | CROARING_AVX512DQ | CROARING_AVX512BW | + CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); +#endif + +#if defined(__x86_64__) || defined(_M_AMD64) // x64 + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { +#if CROARING_REGULAR_VISUAL_STUDIO + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + __asm__("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint64_t xgetbv(void) { +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t xcr0_lo, xcr0_hi; + __asm__("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); +#endif +} + +/** + * This is a relatively expensive function but it will get called at most + * *once* per compilation units. Normally, the CRoaring library is built + * as one compilation unit. + */ +static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { + uint32_t eax, ebx, ecx, edx; + uint32_t host_isa = 0x0; + // Can be found on Intel ISA Reference for CPUID + static uint32_t cpuid_avx2_bit = + 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 + static uint32_t cpuid_bmi1_bit = + 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 + static uint32_t cpuid_bmi2_bit = + 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 + static uint32_t cpuid_avx512f_bit = + 1 << 16; ///< @private bit 16 of EBX for EAX=0x7 + static uint32_t cpuid_avx512dq_bit = + 1 << 17; ///< @private bit 17 of EBX for EAX=0x7 + static uint32_t cpuid_avx512bw_bit = + 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 + static uint32_t cpuid_avx512vbmi2_bit = + 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 + static uint32_t cpuid_avx512bitalg_bit = + 1 << 12; ///< @private bit 12 of ECX for EAX=0x7 + static uint32_t cpuid_avx512vpopcntdq_bit = + 1 << 14; ///< @private bit 14 of ECX for EAX=0x7 + static uint64_t cpuid_avx256_saved = 1 << 2; ///< @private bit 2 = AVX + static uint64_t cpuid_avx512_saved = + 7 << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM + static uint32_t cpuid_sse42_bit = + 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 + static uint32_t cpuid_osxsave = + (1 << 26) | (1 << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + static uint32_t cpuid_pclmulqdq_bit = + 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 + + // EBX for EAX=0x1 + eax = 0x1; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + host_isa |= CROARING_SSE42; + } else { + return host_isa; // everything after is redundant + } + + if (ecx & cpuid_pclmulqdq_bit) { + host_isa |= CROARING_PCLMULQDQ; + } + + if ((ecx & cpuid_osxsave) != cpuid_osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_avx256_saved) == 0) { + return host_isa; + } + + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_avx2_bit) { + host_isa |= CROARING_AVX2; + } + if (ebx & cpuid_bmi1_bit) { + host_isa |= CROARING_BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + host_isa |= CROARING_BMI2; + } + + if (!((xcr0 & cpuid_avx512_saved) == cpuid_avx512_saved)) { + return host_isa; + } + + if (ebx & cpuid_avx512f_bit) { + host_isa |= CROARING_AVX512F; + } + + if (ebx & cpuid_avx512bw_bit) { + host_isa |= CROARING_AVX512BW; + } + + if (ebx & cpuid_avx512dq_bit) { + host_isa |= CROARING_AVX512DQ; + } + + if (ecx & cpuid_avx512vbmi2_bit) { + host_isa |= CROARING_AVX512VBMI2; + } + + if (ecx & cpuid_avx512bitalg_bit) { + host_isa |= CROARING_AVX512BITALG; + } + + if (ecx & cpuid_avx512vpopcntdq_bit) { + host_isa |= CROARING_AVX512VPOPCNTDQ; + } + + return host_isa; +} + +#endif // end SIMD extension detection code + +#if defined(__x86_64__) || defined(_M_AMD64) // x64 + +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP +static inline uint32_t croaring_detect_supported_architectures(void) { + // thread-safe as per the C++11 standard. + static uint32_t buffer = dynamic_croaring_detect_supported_architectures(); + return buffer; +} +#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C +static uint32_t croaring_detect_supported_architectures(void) { + // we use an atomic for thread safety + static _Atomic uint32_t buffer = CROARING_UNINITIALIZED; + if (buffer == CROARING_UNINITIALIZED) { + // atomicity is sufficient + buffer = dynamic_croaring_detect_supported_architectures(); + } + return buffer; +} +#else +// If we do not have atomics, we do the best we can. +static inline uint32_t croaring_detect_supported_architectures(void) { + static uint32_t buffer = CROARING_UNINITIALIZED; + if (buffer == CROARING_UNINITIALIZED) { + buffer = dynamic_croaring_detect_supported_architectures(); + } + return buffer; +} +#endif // CROARING_C_ATOMIC + +#ifdef ROARING_DISABLE_AVX -// We need portability.h to be included first, see -// https://github.com/RoaringBitmap/CRoaring/issues/394 -#if CROARING_REGULAR_VISUAL_STUDIO -#include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) -#include -#endif // CROARING_REGULAR_VISUAL_STUDIO +int croaring_hardware_support(void) { return 0; } -#if CROARING_IS_X64 -#ifndef CROARING_COMPILER_SUPPORTS_AVX512 -#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 -#endif +#elif defined(__AVX512F__) && defined(__AVX512DQ__) && \ + defined(__AVX512BW__) && defined(__AVX512VBMI2__) && \ + defined(__AVX512BITALG__) && defined(__AVX512VPOPCNTDQ__) +int croaring_hardware_support(void) { + return ROARING_SUPPORTS_AVX2 | ROARING_SUPPORTS_AVX512; +} +#elif defined(__AVX2__) -#ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +int croaring_hardware_support(void) { + static +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C + _Atomic #endif -enum croaring_instruction_set { - CROARING_DEFAULT = 0x0, - CROARING_NEON = 0x1, - CROARING_AVX2 = 0x4, - CROARING_SSE42 = 0x8, - CROARING_PCLMULQDQ = 0x10, - CROARING_BMI1 = 0x20, - CROARING_BMI2 = 0x40, - CROARING_ALTIVEC = 0x80, - CROARING_AVX512F = 0x100, - CROARING_AVX512DQ = 0x200, - CROARING_AVX512BW = 0x400, - CROARING_AVX512VBMI2 = 0x800, - CROARING_AVX512BITALG = 0x1000, - CROARING_AVX512VPOPCNTDQ = 0x2000, - CROARING_UNINITIALIZED = 0x8000 -}; + int support = 0xFFFFFFF; + if (support == 0xFFFFFFF) { + bool avx512_support = false; +#if CROARING_COMPILER_SUPPORTS_AVX512 + avx512_support = + ((croaring_detect_supported_architectures() & + CROARING_AVX512_REQUIRED) == CROARING_AVX512_REQUIRED); +#endif + support = ROARING_SUPPORTS_AVX2 | + (avx512_support ? ROARING_SUPPORTS_AVX512 : 0); + } + return support; +} +#else +int croaring_hardware_support(void) { + static +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C + _Atomic +#endif + int support = 0xFFFFFFF; + if (support == 0xFFFFFFF) { + bool has_avx2 = (croaring_detect_supported_architectures() & + CROARING_AVX2) == CROARING_AVX2; + bool has_avx512 = false; #if CROARING_COMPILER_SUPPORTS_AVX512 -unsigned int CROARING_AVX512_REQUIRED = (CROARING_AVX512F | CROARING_AVX512DQ | CROARING_AVX512BW | CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); + has_avx512 = (croaring_detect_supported_architectures() & + CROARING_AVX512_REQUIRED) == CROARING_AVX512_REQUIRED; +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + support = (has_avx2 ? ROARING_SUPPORTS_AVX2 : 0) | + (has_avx512 ? ROARING_SUPPORTS_AVX512 : 0); + } + return support; +} #endif -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +/* end file src/isadetection.c */ +/* begin file src/memory.c */ +#include -static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, - uint32_t *edx) { -#if CROARING_REGULAR_VISUAL_STUDIO - int cpu_info[4]; - __cpuidex(cpu_info, *eax, *ecx); - *eax = cpu_info[0]; - *ebx = cpu_info[1]; - *ecx = cpu_info[2]; - *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) - uint32_t level = *eax; - __get_cpuid(level, eax, ebx, ecx, edx); +// without the following, we get lots of warnings about posix_memalign +#ifndef __cplusplus +extern int posix_memalign(void** __memptr, size_t __alignment, size_t __size); +#endif //__cplusplus // C++ does not have a well defined signature + +// portable version of posix_memalign +static void* roaring_bitmap_aligned_malloc(size_t alignment, size_t size) { + void* p; +#ifdef _MSC_VER + p = _aligned_malloc(size, alignment); +#elif defined(__MINGW32__) || defined(__MINGW64__) + p = __mingw_aligned_malloc(size, alignment); #else - uint32_t a = *eax, b, c = *ecx, d; - __asm__("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); - *eax = a; - *ebx = b; - *ecx = c; - *edx = d; + // somehow, if this is used before including "x86intrin.h", it creates an + // implicit defined warning. + if (posix_memalign(&p, alignment, size) != 0) return NULL; #endif + return p; } - -static inline uint64_t xgetbv(void) { -#if defined(_MSC_VER) - return _xgetbv(0); +static void roaring_bitmap_aligned_free(void* memblock) { +#ifdef _MSC_VER + _aligned_free(memblock); +#elif defined(__MINGW32__) || defined(__MINGW64__) + __mingw_aligned_free(memblock); #else - uint32_t xcr0_lo, xcr0_hi; - __asm__("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); - return xcr0_lo | ((uint64_t)xcr0_hi << 32); + free(memblock); #endif } -/** - * This is a relatively expensive function but it will get called at most - * *once* per compilation units. Normally, the CRoaring library is built - * as one compilation unit. - */ -static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { - uint32_t eax, ebx, ecx, edx; - uint32_t host_isa = 0x0; - // Can be found on Intel ISA Reference for CPUID - static uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 - static uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 - static uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 - static uint32_t cpuid_avx512f_bit = 1 << 16; ///< @private bit 16 of EBX for EAX=0x7 - static uint32_t cpuid_avx512dq_bit = 1 << 17; ///< @private bit 17 of EBX for EAX=0x7 - static uint32_t cpuid_avx512bw_bit = 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 - static uint32_t cpuid_avx512vbmi2_bit = 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 - static uint32_t cpuid_avx512bitalg_bit = 1 << 12; ///< @private bit 12 of ECX for EAX=0x7 - static uint32_t cpuid_avx512vpopcntdq_bit = 1 << 14; ///< @private bit 14 of ECX for EAX=0x7 - static uint64_t cpuid_avx256_saved = 1 << 2; ///< @private bit 2 = AVX - static uint64_t cpuid_avx512_saved = 7 << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM - static uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 - static uint32_t cpuid_osxsave = (1 << 26) | (1 << 27); ///< @private bits 26+27 of ECX for EAX=0x1 - static uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 - - - // EBX for EAX=0x1 - eax = 0x1; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - - if (ecx & cpuid_sse42_bit) { - host_isa |= CROARING_SSE42; - } else { - return host_isa; // everything after is redundant - } +static roaring_memory_t global_memory_hook = { + .malloc = malloc, + .realloc = realloc, + .calloc = calloc, + .free = free, + .aligned_malloc = roaring_bitmap_aligned_malloc, + .aligned_free = roaring_bitmap_aligned_free, +}; - if (ecx & cpuid_pclmulqdq_bit) { - host_isa |= CROARING_PCLMULQDQ; - } +void roaring_init_memory_hook(roaring_memory_t memory_hook) { + global_memory_hook = memory_hook; +} - if ((ecx & cpuid_osxsave) != cpuid_osxsave) { - return host_isa; - } +void* roaring_malloc(size_t n) { return global_memory_hook.malloc(n); } - // xgetbv for checking if the OS saves registers - uint64_t xcr0 = xgetbv(); +void* roaring_realloc(void* p, size_t new_sz) { + return global_memory_hook.realloc(p, new_sz); +} - if ((xcr0 & cpuid_avx256_saved) == 0) { - return host_isa; - } +void* roaring_calloc(size_t n_elements, size_t element_size) { + return global_memory_hook.calloc(n_elements, element_size); +} - // ECX for EAX=0x7 - eax = 0x7; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - if (ebx & cpuid_avx2_bit) { - host_isa |= CROARING_AVX2; - } - if (ebx & cpuid_bmi1_bit) { - host_isa |= CROARING_BMI1; - } +void roaring_free(void* p) { global_memory_hook.free(p); } - if (ebx & cpuid_bmi2_bit) { - host_isa |= CROARING_BMI2; - } +void* roaring_aligned_malloc(size_t alignment, size_t size) { + return global_memory_hook.aligned_malloc(alignment, size); +} - if (!((xcr0 & cpuid_avx512_saved) == cpuid_avx512_saved)) { - return host_isa; - } +void roaring_aligned_free(void* p) { global_memory_hook.aligned_free(p); } +/* end file src/memory.c */ +/* begin file src/roaring.c */ +#include +#include +#include +#include +#include +#include - if (ebx & cpuid_avx512f_bit) { - host_isa |= CROARING_AVX512F; - } - - if (ebx & cpuid_avx512bw_bit) { - host_isa |= CROARING_AVX512BW; - } - - if (ebx & cpuid_avx512dq_bit) { - host_isa |= CROARING_AVX512DQ; - } - - if (ecx & cpuid_avx512vbmi2_bit) { - host_isa |= CROARING_AVX512VBMI2; - } - - if (ecx & cpuid_avx512bitalg_bit) { - host_isa |= CROARING_AVX512BITALG; - } - - if (ecx & cpuid_avx512vpopcntdq_bit) { - host_isa |= CROARING_AVX512VPOPCNTDQ; - } - return host_isa; +// Include after roaring.h + +#ifdef __cplusplus +using namespace ::roaring::internal; + +extern "C" { +namespace roaring { +namespace api { +#endif + +#define CROARING_SERIALIZATION_ARRAY_UINT32 1 +#define CROARING_SERIALIZATION_CONTAINER 2 +extern inline int roaring_trailing_zeroes(unsigned long long input_num); +extern inline int roaring_leading_zeroes(unsigned long long input_num); +extern inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r); +extern inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r); +extern inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, + bool cow); +extern inline roaring_bitmap_t *roaring_bitmap_create(void); +extern inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max); +extern inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, + uint64_t min, uint64_t max); + +static inline bool is_cow(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_COW; +} +static inline bool is_frozen(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_FROZEN; } -#endif // end SIMD extension detection code +// this is like roaring_bitmap_add, but it populates pointer arguments in such a +// way +// that we can recover the container touched, which, in turn can be used to +// accelerate some functions (when you repeatedly need to add to the same +// container) +static inline container_t *containerptr_roaring_bitmap_add(roaring_bitmap_t *r, + uint32_t val, + uint8_t *type, + int *index) { + roaring_array_t *ra = &r->high_low_container; + uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *c = ra_get_container_at_index(ra, (uint16_t)i, type); + uint8_t new_type = *type; + container_t *c2 = container_add(c, val & 0xFFFF, *type, &new_type); + *index = i; + if (c2 != c) { + container_free(c, *type); + ra_set_container_at_index(ra, i, c2, new_type); + *type = new_type; + return c2; + } else { + return c; + } + } else { + array_container_t *new_ac = array_container_create(); + container_t *c = + container_add(new_ac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, type); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(ra, -i - 1, hb, c, *type); + *index = -i - 1; + return c; + } +} -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (!ans) { + return NULL; + } + bool is_ok = ra_init_with_capacity(&ans->high_low_container, cap); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + return ans; +} -#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP -static inline uint32_t croaring_detect_supported_architectures(void) { - // thread-safe as per the C++11 standard. - static uint32_t buffer = dynamic_croaring_detect_supported_architectures(); - return buffer; +bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap) { + return ra_init_with_capacity(&r->high_low_container, cap); } -#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C -static uint32_t croaring_detect_supported_architectures(void) { - // we use an atomic for thread safety - static _Atomic uint32_t buffer = CROARING_UNINITIALIZED; - if (buffer == CROARING_UNINITIALIZED) { - // atomicity is sufficient - buffer = dynamic_croaring_detect_supported_architectures(); + +static inline void add_bulk_impl(roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + uint8_t typecode; + int idx; + context->container = + containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + context->typecode = typecode; + context->idx = idx; + context->key = key; + } else { + // no need to seek the container, it is at hand + // because we already have the container at hand, we can do the + // insertion directly, bypassing the roaring_bitmap_add call + uint8_t new_typecode; + container_t *container2 = container_add( + context->container, val & 0xFFFF, context->typecode, &new_typecode); + if (container2 != context->container) { + // rare instance when we need to change the container type + container_free(context->container, context->typecode); + ra_set_container_at_index(&r->high_low_container, context->idx, + container2, new_typecode); + context->typecode = new_typecode; + context->container = container2; + } } - return buffer; } -#else -// If we do not have atomics, we do the best we can. -static inline uint32_t croaring_detect_supported_architectures(void) { - static uint32_t buffer = CROARING_UNINITIALIZED; - if (buffer == CROARING_UNINITIALIZED) { - buffer = dynamic_croaring_detect_supported_architectures(); + +void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + uint32_t val; + const uint32_t *start = vals; + const uint32_t *end = vals + n_args; + const uint32_t *current_val = start; + + if (n_args == 0) { + return; + } + + uint8_t typecode; + int idx; + container_t *container; + val = *current_val; + container = containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + roaring_bulk_context_t context = {container, idx, (uint16_t)(val >> 16), + typecode}; + + for (; current_val != end; current_val++) { + memcpy(&val, current_val, sizeof(val)); + add_bulk_impl(r, &context, val); + } +} + +void roaring_bitmap_add_bulk(roaring_bitmap_t *r, + roaring_bulk_context_t *context, uint32_t val) { + add_bulk_impl(r, context, val); +} + +bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + int32_t start_idx = -1; + if (context->container != NULL && context->key < key) { + start_idx = context->idx; + } + int idx = ra_advance_until(&r->high_low_container, key, start_idx); + if (idx == ra_get_size(&r->high_low_container)) { + return false; + } + uint8_t typecode; + context->container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)idx, &typecode); + context->typecode = typecode; + context->idx = idx; + context->key = + ra_get_key_at_index(&r->high_low_container, (uint16_t)idx); + // ra_advance_until finds the next key >= the target, we found a later + // container. + if (context->key != key) { + return false; + } } - return buffer; + // context is now set up + return container_contains(context->container, val & 0xFFFF, + context->typecode); } -#endif // CROARING_C_ATOMIC - -#ifdef ROARING_DISABLE_AVX -int croaring_hardware_support(void) { - return 0; +roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals) { + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bitmap_add_many(answer, n_args, vals); + return answer; } -#elif defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VBMI2__) && defined(__AVX512BITALG__) && defined(__AVX512VPOPCNTDQ__) -int croaring_hardware_support(void) { - return ROARING_SUPPORTS_AVX2 | ROARING_SUPPORTS_AVX512; +roaring_bitmap_t *roaring_bitmap_of(size_t n_args, ...) { + // todo: could be greatly optimized but we do not expect this call to ever + // include long lists + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bulk_context_t context = {0}; + va_list ap; + va_start(ap, n_args); + for (size_t i = 0; i < n_args; i++) { + uint32_t val = va_arg(ap, uint32_t); + roaring_bitmap_add_bulk(answer, &context, val); + } + va_end(ap); + return answer; } -#elif defined(__AVX2__) -int croaring_hardware_support(void) { - static int support = 0xFFFFFFF; - if(support == 0xFFFFFFF) { - bool avx512_support = false; -#if CROARING_COMPILER_SUPPORTS_AVX512 - avx512_support = ( (croaring_detect_supported_architectures() & CROARING_AVX512_REQUIRED) - == CROARING_AVX512_REQUIRED); -#endif - support = ROARING_SUPPORTS_AVX2 | (avx512_support ? ROARING_SUPPORTS_AVX512 : 0); - } - return support; +static inline uint64_t minimum_uint64(uint64_t a, uint64_t b) { + return (a < b) ? a : b; } -#else -int croaring_hardware_support(void) { - static int support = 0xFFFFFFF; - if(support == 0xFFFFFFF) { - bool has_avx2 = (croaring_detect_supported_architectures() & CROARING_AVX2) == CROARING_AVX2; - bool has_avx512 = false; -#if CROARING_COMPILER_SUPPORTS_AVX512 - has_avx512 = (croaring_detect_supported_architectures() & CROARING_AVX512_REQUIRED) == CROARING_AVX512_REQUIRED; -#endif // CROARING_COMPILER_SUPPORTS_AVX512 - support = (has_avx2 ? ROARING_SUPPORTS_AVX2 : 0) | (has_avx512 ? ROARING_SUPPORTS_AVX512 : 0); - } - return support; +roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, + uint32_t step) { + if (max >= UINT64_C(0x100000000)) { + max = UINT64_C(0x100000000); + } + if (step == 0) return NULL; + if (max <= min) return NULL; + roaring_bitmap_t *answer = roaring_bitmap_create(); + if (step >= (1 << 16)) { + for (uint32_t value = (uint32_t)min; value < max; value += step) { + roaring_bitmap_add(answer, value); + } + return answer; + } + uint64_t min_tmp = min; + do { + uint32_t key = (uint32_t)min_tmp >> 16; + uint32_t container_min = min_tmp & 0xFFFF; + uint32_t container_max = + (uint32_t)minimum_uint64(max - (key << 16), 1 << 16); + uint8_t type; + container_t *container = container_from_range( + &type, container_min, container_max, (uint16_t)step); + ra_append(&answer->high_low_container, (uint16_t)key, container, type); + uint32_t gap = container_max - container_min + step - 1; + min_tmp += gap - (gap % step); + } while (min_tmp < max); + // cardinality of bitmap will be ((uint64_t) max - min + step - 1 ) / step + return answer; } -#endif -#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 -#ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { -#endif -/* end file src/isadetection.c */ -/* begin file src/memory.c */ -#include +void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { + return; + } -// without the following, we get lots of warnings about posix_memalign -#ifndef __cplusplus -extern int posix_memalign(void **__memptr, size_t __alignment, size_t __size); -#endif //__cplusplus // C++ does not have a well defined signature + roaring_array_t *ra = &r->high_low_container; -// portable version of posix_memalign -static void *roaring_bitmap_aligned_malloc(size_t alignment, size_t size) { - void *p; -#ifdef _MSC_VER - p = _aligned_malloc(size, alignment); -#elif defined(__MINGW32__) || defined(__MINGW64__) - p = __mingw_aligned_malloc(size, alignment); -#else - // somehow, if this is used before including "x86intrin.h", it creates an - // implicit defined warning. - if (posix_memalign(&p, alignment, size) != 0) return NULL; -#endif - return p; -} + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; -static void roaring_bitmap_aligned_free(void *memblock) { -#ifdef _MSC_VER - _aligned_free(memblock); -#elif defined(__MINGW32__) || defined(__MINGW64__) - __mingw_aligned_free(memblock); -#else - free(memblock); -#endif -} + int32_t num_required_containers = max_key - min_key + 1; + int32_t suffix_length = + count_greater(ra->keys, ra->size, (uint16_t)max_key); + int32_t prefix_length = + count_less(ra->keys, ra->size - suffix_length, (uint16_t)min_key); + int32_t common_length = ra->size - prefix_length - suffix_length; -static roaring_memory_t global_memory_hook = { - .malloc = malloc, - .realloc = realloc, - .calloc = calloc, - .free = free, - .aligned_malloc = roaring_bitmap_aligned_malloc, - .aligned_free = roaring_bitmap_aligned_free, -}; + if (num_required_containers > common_length) { + ra_shift_tail(ra, suffix_length, + num_required_containers - common_length); + } -void roaring_init_memory_hook(roaring_memory_t memory_hook) { - global_memory_hook = memory_hook; -} + int32_t src = prefix_length + common_length - 1; + int32_t dst = ra->size - suffix_length - 1; + for (uint32_t key = max_key; key != min_key - 1; + key--) { // beware of min_key==0 + uint32_t container_min = (min_key == key) ? (min & 0xffff) : 0; + uint32_t container_max = (max_key == key) ? (max & 0xffff) : 0xffff; + container_t *new_container; + uint8_t new_type; -void* roaring_malloc(size_t n) { - return global_memory_hook.malloc(n); + if (src >= 0 && ra->keys[src] == key) { + ra_unshare_container_at_index(ra, (uint16_t)src); + new_container = + container_add_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); + } + src--; + } else { + new_container = container_from_range(&new_type, container_min, + container_max + 1, 1); + } + ra_replace_key_and_container_at_index(ra, dst, (uint16_t)key, + new_container, new_type); + dst--; + } } -void* roaring_realloc(void* p, size_t new_sz) { - return global_memory_hook.realloc(p, new_sz); -} +void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { + return; + } -void* roaring_calloc(size_t n_elements, size_t element_size) { - return global_memory_hook.calloc(n_elements, element_size); -} + roaring_array_t *ra = &r->high_low_container; -void roaring_free(void* p) { - global_memory_hook.free(p); + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; + + int32_t src = count_less(ra->keys, ra->size, (uint16_t)min_key); + int32_t dst = src; + while (src < ra->size && ra->keys[src] <= max_key) { + uint32_t container_min = + (min_key == ra->keys[src]) ? (min & 0xffff) : 0; + uint32_t container_max = + (max_key == ra->keys[src]) ? (max & 0xffff) : 0xffff; + ra_unshare_container_at_index(ra, (uint16_t)src); + container_t *new_container; + uint8_t new_type; + new_container = + container_remove_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); + } + if (new_container) { + ra_replace_key_and_container_at_index(ra, dst, ra->keys[src], + new_container, new_type); + dst++; + } + src++; + } + if (src > dst) { + ra_shift_tail(ra, ra->size - src, dst - src); + } } -void* roaring_aligned_malloc(size_t alignment, size_t size) { - return global_memory_hook.aligned_malloc(alignment, size); +void roaring_bitmap_printf(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + printf("{"); + for (int i = 0; i < ra->size; ++i) { + container_printf_as_uint32_array(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + + if (i + 1 < ra->size) { + printf(","); + } + } + printf("}"); } -void roaring_aligned_free(void* p) { - global_memory_hook.aligned_free(p); +void roaring_bitmap_printf_describe(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + printf("{"); + for (int i = 0; i < ra->size; ++i) { + printf("%d: %s (%d)", ra->keys[i], + get_full_container_name(ra->containers[i], ra->typecodes[i]), + container_get_cardinality(ra->containers[i], ra->typecodes[i])); + if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { + printf("(shared count = %" PRIu32 " )", + croaring_refcount_get( + &(CAST_shared(ra->containers[i])->counter))); + } + + if (i + 1 < ra->size) { + printf(", "); + } + } + printf("}"); } -/* end file src/memory.c */ -/* begin file src/roaring.c */ -#include -#include -#include -#include -#include -#include +typedef struct min_max_sum_s { + uint32_t min; + uint32_t max; + uint64_t sum; +} min_max_sum_t; +static bool min_max_sum_fnc(uint32_t value, void *param) { + min_max_sum_t *mms = (min_max_sum_t *)param; + if (value > mms->max) mms->max = value; + if (value < mms->min) mms->min = value; + mms->sum += value; + return true; // we always process all data points +} -#ifdef __cplusplus -using namespace ::roaring::internal; +/** + * (For advanced users.) + * Collect statistics about the bitmap + */ +void roaring_bitmap_statistics(const roaring_bitmap_t *r, + roaring_statistics_t *stat) { + const roaring_array_t *ra = &r->high_low_container; -extern "C" { namespace roaring { namespace api { -#endif + memset(stat, 0, sizeof(*stat)); + stat->n_containers = ra->size; + stat->cardinality = roaring_bitmap_get_cardinality(r); + min_max_sum_t mms; + mms.min = UINT32_C(0xFFFFFFFF); + mms.max = UINT32_C(0); + mms.sum = 0; + roaring_iterate(r, &min_max_sum_fnc, &mms); + stat->min_value = mms.min; + stat->max_value = mms.max; + stat->sum_value = mms.sum; -#define CROARING_SERIALIZATION_ARRAY_UINT32 1 -#define CROARING_SERIALIZATION_CONTAINER 2 + for (int i = 0; i < ra->size; ++i) { + uint8_t truetype = + get_container_type(ra->containers[i], ra->typecodes[i]); + uint32_t card = + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + uint32_t sbytes = + container_size_in_bytes(ra->containers[i], ra->typecodes[i]); + switch (truetype) { + case BITSET_CONTAINER_TYPE: + stat->n_bitset_containers++; + stat->n_values_bitset_containers += card; + stat->n_bytes_bitset_containers += sbytes; + break; + case ARRAY_CONTAINER_TYPE: + stat->n_array_containers++; + stat->n_values_array_containers += card; + stat->n_bytes_array_containers += sbytes; + break; + case RUN_CONTAINER_TYPE: + stat->n_run_containers++; + stat->n_values_run_containers += card; + stat->n_bytes_run_containers += sbytes; + break; + default: + assert(false); + roaring_unreachable; + } + } +} -extern inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r); -extern inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t* r); -extern inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t* r, bool cow); -extern inline roaring_bitmap_t *roaring_bitmap_create(void); -extern inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, uint64_t max); -extern inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, uint64_t min, uint64_t max); +/* + * Checks that: + * - Array containers are sorted and contain no duplicates + * - Range containers are sorted and contain no overlapping ranges + * - Roaring containers are sorted by key and there are no duplicate keys + * - The correct container type is use for each container (e.g. bitmaps aren't + * used for small containers) + */ +bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, + const char **reason) { + const char *reason_local; + if (reason == NULL) { + // Always allow assigning through *reason + reason = &reason_local; + } + *reason = NULL; + const roaring_array_t *ra = &r->high_low_container; + if (ra->size < 0) { + *reason = "negative size"; + return false; + } + if (ra->allocation_size < 0) { + *reason = "negative allocation size"; + return false; + } + if (ra->size > ra->allocation_size) { + *reason = "more containers than allocated space"; + return false; + } + if (ra->flags & ~(ROARING_FLAG_COW | ROARING_FLAG_FROZEN)) { + *reason = "invalid flags"; + return false; + } + if (ra->size == 0) { + return true; + } -static inline bool is_cow(const roaring_bitmap_t *r) { - return r->high_low_container.flags & ROARING_FLAG_COW; -} -static inline bool is_frozen(const roaring_bitmap_t *r) { - return r->high_low_container.flags & ROARING_FLAG_FROZEN; -} + if (ra->keys == NULL) { + *reason = "keys is NULL"; + return false; + } + if (ra->typecodes == NULL) { + *reason = "typecodes is NULL"; + return false; + } + if (ra->containers == NULL) { + *reason = "containers is NULL"; + return false; + } -// this is like roaring_bitmap_add, but it populates pointer arguments in such a -// way -// that we can recover the container touched, which, in turn can be used to -// accelerate some functions (when you repeatedly need to add to the same -// container) -static inline container_t *containerptr_roaring_bitmap_add( - roaring_bitmap_t *r, uint32_t val, - uint8_t *type, int *index -){ - roaring_array_t *ra = &r->high_low_container; + uint32_t prev_key = ra->keys[0]; + for (int32_t i = 1; i < ra->size; ++i) { + if (ra->keys[i] <= prev_key) { + *reason = "keys not strictly increasing"; + return false; + } + prev_key = ra->keys[i]; + } - uint16_t hb = val >> 16; - const int i = ra_get_index(ra, hb); - if (i >= 0) { - ra_unshare_container_at_index(ra, i); - container_t *c = ra_get_container_at_index(ra, i, type); - uint8_t new_type = *type; - container_t *c2 = container_add(c, val & 0xFFFF, *type, &new_type); - *index = i; - if (c2 != c) { - container_free(c, *type); - ra_set_container_at_index(ra, i, c2, new_type); - *type = new_type; - return c2; - } else { - return c; + for (int32_t i = 0; i < ra->size; ++i) { + if (!container_internal_validate(ra->containers[i], ra->typecodes[i], + reason)) { + // reason should already be set + if (*reason == NULL) { + *reason = "container failed to validate but no reason given"; + } + return false; } - } else { - array_container_t *new_ac = array_container_create(); - container_t *c = container_add(new_ac, val & 0xFFFF, - ARRAY_CONTAINER_TYPE, type); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(ra, -i - 1, hb, c, *type); - *index = -i - 1; - return c; } + + return true; } -roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap) { +roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r) { roaring_bitmap_t *ans = (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); if (!ans) { return NULL; } - bool is_ok = ra_init_with_capacity(&ans->high_low_container, cap); - if (!is_ok) { + if (!ra_init_with_capacity( // allocation of list of containers can fail + &ans->high_low_container, r->high_low_container.size)) { roaring_free(ans); return NULL; } + if (!ra_overwrite( // memory allocation of individual containers may fail + &r->high_low_container, &ans->high_low_container, is_cow(r))) { + roaring_bitmap_free(ans); // overwrite should leave in freeable state + return NULL; + } + roaring_bitmap_set_copy_on_write(ans, is_cow(r)); return ans; } -bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap) { - return ra_init_with_capacity(&r->high_low_container, cap); -} - -static inline void add_bulk_impl(roaring_bitmap_t *r, - roaring_bulk_context_t *context, - uint32_t val) { - uint16_t key = val >> 16; - if (context->container == NULL || context->key != key) { - uint8_t typecode; - int idx; - context->container = containerptr_roaring_bitmap_add( - r, val, &typecode, &idx); - context->typecode = typecode; - context->idx = idx; - context->key = key; - } else { - // no need to seek the container, it is at hand - // because we already have the container at hand, we can do the - // insertion directly, bypassing the roaring_bitmap_add call - uint8_t new_typecode; - container_t *container2 = container_add( - context->container, val & 0xFFFF, context->typecode, &new_typecode); - if (container2 != context->container) { - // rare instance when we need to change the container type - container_free(context->container, context->typecode); - ra_set_container_at_index(&r->high_low_container, context->idx, - container2, new_typecode); - context->typecode = new_typecode; - context->container = container2; - } - } +bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, + const roaring_bitmap_t *src) { + roaring_bitmap_set_copy_on_write(dest, is_cow(src)); + return ra_overwrite(&src->high_low_container, &dest->high_low_container, + is_cow(src)); } -void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, - const uint32_t *vals) { - uint32_t val; - const uint32_t *start = vals; - const uint32_t *end = vals + n_args; - const uint32_t *current_val = start; - - if (n_args == 0) { +void roaring_bitmap_free(const roaring_bitmap_t *r) { + if (r == NULL) { return; } - - uint8_t typecode; - int idx; - container_t *container; - val = *current_val; - container = containerptr_roaring_bitmap_add(r, val, &typecode, &idx); - roaring_bulk_context_t context = {container, idx, (uint16_t)(val >> 16), typecode}; - - for (; current_val != end; current_val++) { - memcpy(&val, current_val, sizeof(val)); - add_bulk_impl(r, &context, val); - } -} - -void roaring_bitmap_add_bulk(roaring_bitmap_t *r, - roaring_bulk_context_t *context, uint32_t val) { - add_bulk_impl(r, context, val); -} - -bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, - roaring_bulk_context_t *context, - uint32_t val) -{ - uint16_t key = val >> 16; - if (context->container == NULL || context->key != key) { - int32_t start_idx = -1; - if (context->container != NULL && context->key < key) { - start_idx = context->idx; - } - int idx = ra_advance_until(&r->high_low_container, key, start_idx); - if (idx == ra_get_size(&r->high_low_container)) { - return false; - } - uint8_t typecode; - context->container = ra_get_container_at_index(&r->high_low_container, idx, &typecode); - context->typecode = typecode; - context->idx = idx; - context->key = ra_get_key_at_index(&r->high_low_container, idx); - // ra_advance_until finds the next key >= the target, we found a later container. - if (context->key != key) { - return false; - } - } - // context is now set up - return container_contains(context->container, val & 0xFFFF, context->typecode); -} - -roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals) { - roaring_bitmap_t *answer = roaring_bitmap_create(); - roaring_bitmap_add_many(answer, n_args, vals); - return answer; -} - -roaring_bitmap_t *roaring_bitmap_of(size_t n_args, ...) { - // todo: could be greatly optimized but we do not expect this call to ever - // include long lists - roaring_bitmap_t *answer = roaring_bitmap_create(); - roaring_bulk_context_t context = {0}; - va_list ap; - va_start(ap, n_args); - for (size_t i = 0; i < n_args; i++) { - uint32_t val = va_arg(ap, uint32_t); - roaring_bitmap_add_bulk(answer, &context, val); + if (!is_frozen(r)) { + ra_clear((roaring_array_t *)&r->high_low_container); } - va_end(ap); - return answer; + roaring_free((roaring_bitmap_t *)r); } -static inline uint32_t minimum_uint32(uint32_t a, uint32_t b) { - return (a < b) ? a : b; +void roaring_bitmap_clear(roaring_bitmap_t *r) { + ra_reset(&r->high_low_container); } -static inline uint64_t minimum_uint64(uint64_t a, uint64_t b) { - return (a < b) ? a : b; -} +void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t val) { + roaring_array_t *ra = &r->high_low_container; -roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, - uint32_t step) { - if(max >= UINT64_C(0x100000000)) { - max = UINT64_C(0x100000000); - } - if (step == 0) return NULL; - if (max <= min) return NULL; - roaring_bitmap_t *answer = roaring_bitmap_create(); - if (step >= (1 << 16)) { - for (uint32_t value = (uint32_t)min; value < max; value += step) { - roaring_bitmap_add(answer, value); + const uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *container = + ra_get_container_at_index(ra, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); } - return answer; + } else { + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); } - uint64_t min_tmp = min; - do { - uint32_t key = (uint32_t)min_tmp >> 16; - uint32_t container_min = min_tmp & 0xFFFF; - uint32_t container_max = (uint32_t)minimum_uint64(max - (key << 16), 1 << 16); - uint8_t type; - container_t *container = container_from_range(&type, container_min, - container_max, (uint16_t)step); - ra_append(&answer->high_low_container, key, container, type); - uint32_t gap = container_max - container_min + step - 1; - min_tmp += gap - (gap % step); - } while (min_tmp < max); - // cardinality of bitmap will be ((uint64_t) max - min + step - 1 ) / step - return answer; } -void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, uint32_t max) { - if (min > max) { - return; - } - - roaring_array_t *ra = &r->high_low_container; +bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); - uint32_t min_key = min >> 16; - uint32_t max_key = max >> 16; + const int oldCardinality = + container_get_cardinality(container, typecode); - int32_t num_required_containers = max_key - min_key + 1; - int32_t suffix_length = count_greater(ra->keys, ra->size, max_key); - int32_t prefix_length = count_less(ra->keys, ra->size - suffix_length, - min_key); - int32_t common_length = ra->size - prefix_length - suffix_length; + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + result = true; + } else { + const int newCardinality = + container_get_cardinality(container, newtypecode); - if (num_required_containers > common_length) { - ra_shift_tail(ra, suffix_length, - num_required_containers - common_length); + result = oldCardinality != newCardinality; + } + } else { + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); + result = true; } - int32_t src = prefix_length + common_length - 1; - int32_t dst = ra->size - suffix_length - 1; - for (uint32_t key = max_key; key != min_key-1; key--) { // beware of min_key==0 - uint32_t container_min = (min_key == key) ? (min & 0xffff) : 0; - uint32_t container_max = (max_key == key) ? (max & 0xffff) : 0xffff; - container_t* new_container; - uint8_t new_type; + return result; +} - if (src >= 0 && ra->keys[src] == key) { - ra_unshare_container_at_index(ra, src); - new_container = container_add_range(ra->containers[src], - ra->typecodes[src], - container_min, container_max, - &new_type); - if (new_container != ra->containers[src]) { - container_free(ra->containers[src], - ra->typecodes[src]); - } - src--; +void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + if (container_get_cardinality(container2, newtypecode) != 0) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); } else { - new_container = container_from_range(&new_type, container_min, - container_max+1, 1); + ra_remove_at_index_and_free(&r->high_low_container, i); } - ra_replace_key_and_container_at_index(ra, dst, key, new_container, - new_type); - dst--; } } -void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, uint32_t max) { - if (min > max) { - return; - } - - roaring_array_t *ra = &r->high_low_container; +bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); - uint32_t min_key = min >> 16; - uint32_t max_key = max >> 16; + const int oldCardinality = + container_get_cardinality(container, typecode); - int32_t src = count_less(ra->keys, ra->size, min_key); - int32_t dst = src; - while (src < ra->size && ra->keys[src] <= max_key) { - uint32_t container_min = (min_key == ra->keys[src]) ? (min & 0xffff) : 0; - uint32_t container_max = (max_key == ra->keys[src]) ? (max & 0xffff) : 0xffff; - ra_unshare_container_at_index(ra, src); - container_t *new_container; - uint8_t new_type; - new_container = container_remove_range(ra->containers[src], - ra->typecodes[src], - container_min, container_max, - &new_type); - if (new_container != ra->containers[src]) { - container_free(ra->containers[src], - ra->typecodes[src]); - } - if (new_container) { - ra_replace_key_and_container_at_index(ra, dst, ra->keys[src], - new_container, new_type); - dst++; + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); } - src++; - } - if (src > dst) { - ra_shift_tail(ra, ra->size - src, dst - src); - } -} -void roaring_bitmap_printf(const roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; - - printf("{"); - for (int i = 0; i < ra->size; ++i) { - container_printf_as_uint32_array(ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); + const int newCardinality = + container_get_cardinality(container2, newtypecode); - if (i + 1 < ra->size) { - printf(","); + if (newCardinality != 0) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } else { + ra_remove_at_index_and_free(&r->high_low_container, i); } + + result = oldCardinality != newCardinality; } - printf("}"); + return result; } -void roaring_bitmap_printf_describe(const roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; - - printf("{"); - for (int i = 0; i < ra->size; ++i) { - printf("%d: %s (%d)", ra->keys[i], - get_full_container_name(ra->containers[i], ra->typecodes[i]), - container_get_cardinality(ra->containers[i], ra->typecodes[i])); - if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { - printf("(shared count = %" PRIu32 " )", - croaring_refcount_get( - &(CAST_shared(ra->containers[i])->counter))); +void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + if (n_args == 0 || r->high_low_container.size == 0) { + return; + } + int32_t pos = + -1; // position of the container used in the previous iteration + for (size_t i = 0; i < n_args; i++) { + uint16_t key = (uint16_t)(vals[i] >> 16); + if (pos < 0 || key != r->high_low_container.keys[pos]) { + pos = ra_get_index(&r->high_low_container, key); } - - if (i + 1 < ra->size) { - printf(", "); + if (pos >= 0) { + uint8_t new_typecode; + container_t *new_container; + new_container = container_remove( + r->high_low_container.containers[pos], vals[i] & 0xffff, + r->high_low_container.typecodes[pos], &new_typecode); + if (new_container != r->high_low_container.containers[pos]) { + container_free(r->high_low_container.containers[pos], + r->high_low_container.typecodes[pos]); + ra_replace_key_and_container_at_index(&r->high_low_container, + pos, key, new_container, + new_typecode); + } + if (!container_nonzero_cardinality(new_container, new_typecode)) { + container_free(new_container, new_typecode); + ra_remove_at_index(&r->high_low_container, pos); + pos = -1; + } } } - printf("}"); } -typedef struct min_max_sum_s { - uint32_t min; - uint32_t max; - uint64_t sum; -} min_max_sum_t; +// there should be some SIMD optimizations possible here +roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint32_t neededcap = length1 > length2 ? length2 : length1; + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); -static bool min_max_sum_fnc(uint32_t value, void *param) { - min_max_sum_t *mms = (min_max_sum_t *)param; - if (value > mms->max) mms->max = value; - if (value < mms->min) mms->min = value; - mms->sum += value; - return true; // we always process all data points -} + int pos1 = 0, pos2 = 0; -/** -* (For advanced users.) -* Collect statistics about the bitmap -*/ -void roaring_bitmap_statistics(const roaring_bitmap_t *r, - roaring_statistics_t *stat) { - const roaring_array_t *ra = &r->high_low_container; + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - memset(stat, 0, sizeof(*stat)); - stat->n_containers = ra->size; - stat->cardinality = roaring_bitmap_get_cardinality(r); - min_max_sum_t mms; - mms.min = UINT32_C(0xFFFFFFFF); - mms.max = UINT32_C(0); - mms.sum = 0; - roaring_iterate(r, &min_max_sum_fnc, &mms); - stat->min_value = mms.min; - stat->max_value = mms.max; - stat->sum_value = mms.sum; + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_and(c1, type1, c2, type2, &result_type); - for (int i = 0; i < ra->size; ++i) { - uint8_t truetype = - get_container_type(ra->containers[i], ra->typecodes[i]); - uint32_t card = - container_get_cardinality(ra->containers[i], ra->typecodes[i]); - uint32_t sbytes = - container_size_in_bytes(ra->containers[i], ra->typecodes[i]); - switch (truetype) { - case BITSET_CONTAINER_TYPE: - stat->n_bitset_containers++; - stat->n_values_bitset_containers += card; - stat->n_bytes_bitset_containers += sbytes; - break; - case ARRAY_CONTAINER_TYPE: - stat->n_array_containers++; - stat->n_values_array_containers += card; - stat->n_bytes_array_containers += sbytes; - break; - case RUN_CONTAINER_TYPE: - stat->n_run_containers++; - stat->n_values_run_containers += card; - stat->n_bytes_run_containers += sbytes; - break; - default: - assert(false); - roaring_unreachable; + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); // otherwise: memory leak! + } + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } } + return answer; } -/* - * Checks that: - * - Array containers are sorted and contain no duplicates - * - Range containers are sorted and contain no overlapping ranges - * - Roaring containers are sorted by key and there are no duplicate keys - * - The correct container type is use for each container (e.g. bitmaps aren't used for small containers) +/** + * Compute the union of 'number' bitmaps. */ -bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, const char **reason) { - const char *reason_local; - if (reason == NULL) { - // Always allow assigning through *reason - reason = &reason_local; +roaring_bitmap_t *roaring_bitmap_or_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); } - *reason = NULL; - const roaring_array_t *ra = &r->high_low_container; - if (ra->size < 0) { - *reason = "negative size"; - return false; + if (number == 1) { + return roaring_bitmap_copy(x[0]); } - if (ra->allocation_size < 0) { - *reason = "negative allocation size"; - return false; + roaring_bitmap_t *answer = + roaring_bitmap_lazy_or(x[0], x[1], LAZY_OR_BITSET_CONVERSION); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_or_inplace(answer, x[i], LAZY_OR_BITSET_CONVERSION); } - if (ra->size > ra->allocation_size) { - *reason = "more containers than allocated space"; - return false; + roaring_bitmap_repair_after_lazy(answer); + return answer; +} + +/** + * Compute the xor of 'number' bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); } - if (ra->flags & ~(ROARING_FLAG_COW | ROARING_FLAG_FROZEN)) { - *reason = "invalid flags"; - return false; + if (number == 1) { + return roaring_bitmap_copy(x[0]); } - if (ra->size == 0) { - return true; + roaring_bitmap_t *answer = roaring_bitmap_lazy_xor(x[0], x[1]); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_xor_inplace(answer, x[i]); } + roaring_bitmap_repair_after_lazy(answer); + return answer; +} - if (ra->keys == NULL) { - *reason = "keys is NULL"; - return false; - } - if (ra->typecodes == NULL) { - *reason = "typecodes is NULL"; - return false; +// inplace and (modifies its first argument). +void roaring_bitmap_and_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + if (x1 == x2) return; + int pos1 = 0, pos2 = 0, intersection_size = 0; + const int length1 = ra_get_size(&x1->high_low_container); + const int length2 = ra_get_size(&x2->high_low_container); + + // any skipped-over or newly emptied containers in x1 + // have to be freed. + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2, result_type; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_and(c1, type1, c2, type2, &result_type) + : container_iand(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, and + // we need to free the old one + container_free(c1, type1); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_replace_key_and_container_at_index(&x1->high_low_container, + intersection_size, s1, c, + result_type); + intersection_size++; + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + } else if (s1 < s2) { + pos1 = ra_advance_until_freeing(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } } - if (ra->containers == NULL) { - *reason = "containers is NULL"; - return false; + + // if we ended early because x2 ran out, then all remaining in x1 should be + // freed + while (pos1 < length1) { + container_free(x1->high_low_container.containers[pos1], + x1->high_low_container.typecodes[pos1]); + ++pos1; } - uint32_t prev_key = ra->keys[0]; - for (int32_t i = 1; i < ra->size; ++i) { - if (ra->keys[i] <= prev_key) { - *reason = "keys not strictly increasing"; - return false; - } - prev_key = ra->keys[i]; + // all containers after this have either been copied or freed + ra_downsize(&x1->high_low_container, intersection_size); +} + +roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_or(c1, type1, c2, type2, &result_type); - for (int32_t i = 0; i < ra->size; ++i) { - if (!container_internal_validate(ra->containers[i], ra->typecodes[i], reason)) { - // reason should already be set - if (*reason == NULL) { - *reason = "container failed to validate but no reason given"; + // since we assume that the initial containers are non-empty, the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + // c1 = container_clone(c1, type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); } - return false; + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // c2 = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } - - return true; + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; } -roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r) { - roaring_bitmap_t *ans = - (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); - if (!ans) { - return NULL; +// inplace or (modifies its first argument). +void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; } - if (!ra_init_with_capacity( // allocation of list of containers can fail - &ans->high_low_container, r->high_low_container.size) - ){ - roaring_free(ans); - return NULL; + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + + // container_t *c2_clone = container_clone(c2, type2); + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } } - if (!ra_overwrite( // memory allocation of individual containers may fail - &r->high_low_container, &ans->high_low_container, is_cow(r)) - ){ - roaring_bitmap_free(ans); // overwrite should leave in freeable state - return NULL; + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); } - roaring_bitmap_set_copy_on_write(ans, is_cow(r)); - return ans; -} - -bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, - const roaring_bitmap_t *src) { - roaring_bitmap_set_copy_on_write(dest, is_cow(src)); - return ra_overwrite(&src->high_low_container, &dest->high_low_container, - is_cow(src)); } -void roaring_bitmap_free(const roaring_bitmap_t *r) { - if(r == NULL) { return; } - if (!is_frozen(r)) { - ra_clear((roaring_array_t*)&r->high_low_container); +roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); } - roaring_free((roaring_bitmap_t*)r); -} + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_xor(c1, type1, c2, type2, &result_type); -void roaring_bitmap_clear(roaring_bitmap_t *r) { - ra_reset(&r->high_low_container); -} + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); -void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t val) { - roaring_array_t *ra = &r->high_low_container; + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - const uint16_t hb = val >> 16; - const int i = ra_get_index(ra, hb); - uint8_t typecode; - if (i >= 0) { - ra_unshare_container_at_index(ra, i); - container_t *container = - ra_get_container_at_index(ra, i, &typecode); - uint8_t newtypecode = typecode; - container_t *container2 = - container_add(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - } else { - array_container_t *newac = array_container_create(); - container_t *container = container_add(newac, val & 0xFFFF, - ARRAY_CONTAINER_TYPE, &typecode); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, - container, typecode); } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; } -bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - bool result = false; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, i); - container_t *container = - ra_get_container_at_index(&r->high_low_container, i, &typecode); +// inplace xor (modifies its first argument). - const int oldCardinality = - container_get_cardinality(container, typecode); +void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; - uint8_t newtypecode = typecode; - container_t *container2 = - container_add(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - result = true; - } else { - const int newCardinality = - container_get_cardinality(container, newtypecode); + if (0 == length2) return; - result = oldCardinality != newCardinality; - } - } else { - array_container_t *newac = array_container_create(); - container_t *container = container_add(newac, val & 0xFFFF, - ARRAY_CONTAINER_TYPE, &typecode); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, - container, typecode); - result = true; + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; } - return result; -} - -void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, i); - container_t *container = - ra_get_container_at_index(&r->high_low_container, i, &typecode); - uint8_t newtypecode = typecode; - container_t *container2 = - container_remove(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } - if (container_get_cardinality(container2, newtypecode) != 0) { - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } else { - ra_remove_at_index_and_free(&r->high_low_container, i); - } - } -} + // XOR can have new containers inserted from x2, but can also + // lose containers when x1 and x2 are nonempty and identical. -bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - bool result = false; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, i); - container_t *container = - ra_get_container_at_index(&r->high_low_container, i, &typecode); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); - const int oldCardinality = - container_get_cardinality(container, typecode); + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. - uint8_t newtypecode = typecode; - container_t *container2 = - container_remove(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // so release + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } - const int newCardinality = - container_get_cardinality(container2, newtypecode); + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + ra_remove_at_index(&x1->high_low_container, pos1); + --length1; + } - if (newCardinality != 0) { - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } else { - ra_remove_at_index_and_free(&r->high_low_container, i); - } + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - result = oldCardinality != newCardinality; - } - return result; -} + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); -void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, - const uint32_t *vals) { - if (n_args == 0 || r->high_low_container.size == 0) { - return; - } - int32_t pos = -1; // position of the container used in the previous iteration - for (size_t i = 0; i < n_args; i++) { - uint16_t key = (uint16_t)(vals[i] >> 16); - if (pos < 0 || key != r->high_low_container.keys[pos]) { - pos = ra_get_index(&r->high_low_container, key); - } - if (pos >= 0) { - uint8_t new_typecode; - container_t *new_container; - new_container = container_remove(r->high_low_container.containers[pos], - vals[i] & 0xffff, - r->high_low_container.typecodes[pos], - &new_typecode); - if (new_container != r->high_low_container.containers[pos]) { - container_free(r->high_low_container.containers[pos], - r->high_low_container.typecodes[pos]); - ra_replace_key_and_container_at_index(&r->high_low_container, - pos, key, new_container, - new_typecode); - } - if (!container_nonzero_cardinality(new_container, new_typecode)) { - container_free(new_container, new_typecode); - ra_remove_at_index(&r->high_low_container, pos); - pos = -1; + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); } + + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } } -// there should be some SIMD optimizations possible here -roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { +roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { uint8_t result_type = 0; const int length1 = x1->high_low_container.size, length2 = x2->high_low_container.size; - uint32_t neededcap = length1 > length2 ? length2 : length1; - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + if (0 == length1) { + roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(empty_bitmap, + is_cow(x1) || is_cow(x2)); + return empty_bitmap; + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); int pos1 = 0, pos2 = 0; - - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - const uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint8_t type1, type2; + uint16_t s1 = 0; + uint16_t s2 = 0; + while (true) { + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_and(c1, type1, c2, type2, &result_type); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_andnot(c1, type1, c2, type2, &result_type); if (container_nonzero_cardinality(c, result_type)) { ra_append(&answer->high_low_container, s1, c, result_type); } else { - container_free(c, result_type); // otherwise: memory leak! + container_free(c, result_type); } ++pos1; ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + const int next_pos1 = + ra_advance_until(&x1->high_low_container, s2, pos1); + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, next_pos1, + is_cow(x1)); + // TODO : perhaps some of the copy_on_write should be based on + // answer rather than x1 (more stringent?). Many similar cases + pos1 = next_pos1; + if (pos1 == length1) break; } else { // s1 > s2 pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + if (pos2 == length2) break; } } - return answer; -} - -/** - * Compute the union of 'number' bitmaps. - */ -roaring_bitmap_t *roaring_bitmap_or_many(size_t number, - const roaring_bitmap_t **x) { - if (number == 0) { - return roaring_bitmap_create(); - } - if (number == 1) { - return roaring_bitmap_copy(x[0]); - } - roaring_bitmap_t *answer = - roaring_bitmap_lazy_or(x[0], x[1], LAZY_OR_BITSET_CONVERSION); - for (size_t i = 2; i < number; i++) { - roaring_bitmap_lazy_or_inplace(answer, x[i], LAZY_OR_BITSET_CONVERSION); + if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); } - roaring_bitmap_repair_after_lazy(answer); return answer; } -/** - * Compute the xor of 'number' bitmaps. - */ -roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, - const roaring_bitmap_t **x) { - if (number == 0) { - return roaring_bitmap_create(); - } - if (number == 1) { - return roaring_bitmap_copy(x[0]); - } - roaring_bitmap_t *answer = roaring_bitmap_lazy_xor(x[0], x[1]); - for (size_t i = 2; i < number; i++) { - roaring_bitmap_lazy_xor_inplace(answer, x[i]); - } - roaring_bitmap_repair_after_lazy(answer); - return answer; -} +// inplace andnot (modifies its first argument). -// inplace and (modifies its first argument). -void roaring_bitmap_and_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - if (x1 == x2) return; - int pos1 = 0, pos2 = 0, intersection_size = 0; - const int length1 = ra_get_size(&x1->high_low_container); - const int length2 = ra_get_size(&x2->high_low_container); +void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); - // any skipped-over or newly emptied containers in x1 - // have to be freed. - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - const uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + int intersection_size = 0; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_clear(x1); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { if (s1 == s2) { - uint8_t type1, type2, result_type; - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - - // We do the computation "in place" only when c1 is not a shared container. - // Rationale: using a shared container safely with in place computation would - // require making a copy and then doing the computation in place which is likely - // less efficient than avoiding in place entirely and always generating a new - // container. - container_t *c = - (type1 == SHARED_CONTAINER_TYPE) - ? container_and(c1, type1, c2, type2, &result_type) - : container_iand(c1, type1, c2, type2, &result_type); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); - if (c != c1) { // in this instance a new container was created, and - // we need to free the old one - container_free(c1, type1); + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_andnot(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_iandnot(c1, type1, c2, type2, &result_type); } + if (container_nonzero_cardinality(c, result_type)) { ra_replace_key_and_container_at_index(&x1->high_low_container, - intersection_size, s1, c, - result_type); - intersection_size++; + intersection_size++, s1, + c, result_type); } else { container_free(c, result_type); } + ++pos1; ++pos2; - } else if (s1 < s2) { - pos1 = ra_advance_until_freeing(&x1->high_low_container, s2, pos1); + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + if (pos1 != intersection_size) { + container_t *c1 = ra_get_container_at_index( + &x1->high_low_container, (uint16_t)pos1, &type1); + + ra_replace_key_and_container_at_index( + &x1->high_low_container, intersection_size, s1, c1, type1); + } + intersection_size++; + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + } else { // s1 > s2 pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } - // if we ended early because x2 ran out, then all remaining in x1 should be - // freed - while (pos1 < length1) { - container_free(x1->high_low_container.containers[pos1], - x1->high_low_container.typecodes[pos1]); - ++pos1; + if (pos1 < length1) { + // all containers between intersection_size and + // pos1 are junk. However, they have either been moved + // (thus still referenced) or involved in an iandnot + // that will clean up all containers that could not be reused. + // Thus we should not free the junk containers between + // intersection_size and pos1. + if (pos1 > intersection_size) { + // left slide of remaining items + ra_copy_range(&x1->high_low_container, pos1, length1, + intersection_size); + } + // else current placement is fine + intersection_size += (length1 - pos1); } - - // all containers after this have either been copied or freed ra_downsize(&x1->high_low_container, intersection_size); } -roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); +uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + uint64_t card = 0; + for (int i = 0; i < ra->size; ++i) + card += container_get_cardinality(ra->containers[i], ra->typecodes[i]); + return card; +} + +uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, + uint64_t range_start, + uint64_t range_end) { + const roaring_array_t *ra = &r->high_low_container; + + if (range_end > UINT32_MAX) { + range_end = UINT32_MAX + UINT64_C(1); } - if (0 == length2) { - return roaring_bitmap_copy(x1); + if (range_start >= range_end) { + return 0; } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_or(c1, type1, c2, type2, &result_type); + range_end--; // make range_end inclusive + // now we have: 0 <= range_start <= range_end <= UINT32_MAX - // since we assume that the initial containers are non-empty, the - // result here - // can only be non-empty - ra_append(&answer->high_low_container, s1, c, result_type); - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint16_t minhb = (uint16_t)(range_start >> 16); + uint16_t maxhb = (uint16_t)(range_end >> 16); - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - // c1 = container_clone(c1, type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); + uint64_t card = 0; - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - // c2 = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + int i = ra_get_index(ra, minhb); + if (i >= 0) { + if (minhb == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + } else { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + } + if ((range_start & 0xffff) != 0) { + card -= container_rank(ra->containers[i], ra->typecodes[i], + (range_start & 0xffff) - 1); + } + i++; + } else { + i = -i - 1; + } + + for (; i < ra->size; i++) { + uint16_t key = ra->keys[i]; + if (key < maxhb) { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + } else if (key == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + break; + } else { + break; } } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; + + return card; } -// inplace or (modifies its first argument). -void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +bool roaring_bitmap_is_empty(const roaring_bitmap_t *r) { + return r->high_low_container.size == 0; +} - if (0 == length2) return; +void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans) { + ra_to_uint32_array(&r->high_low_container, ans); +} - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; +bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, + size_t limit, uint32_t *ans) { + return ra_range_uint32_array(&r->high_low_container, offset, limit, ans); +} + +/** convert array and bitmap containers to run containers when it is more + * efficient; + * also convert from run containers when more space efficient. Returns + * true if the result has at least one run container. + */ +bool roaring_bitmap_run_optimize(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + ra_unshare_container_at_index( + &r->high_low_container, + (uint16_t)i); // TODO: this introduces extra cloning! + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + container_t *c1 = convert_run_optimize(c, type_original, &type_after); + if (type_after == RUN_CONTAINER_TYPE) { + answer = true; + } + ra_set_container_at_index(&r->high_low_container, i, c1, type_after); } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - if (!container_is_full(c1, type1)) { - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = - (type1 == SHARED_CONTAINER_TYPE) - ? container_or(c1, type1, c2, type2, &result_type) - : container_ior(c1, type1, c2, type2, &result_type); + return answer; +} - if (c != c1) { // in this instance a new container was created, - // and we need to free the old one - container_free(c1, type1); - } - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r) { + size_t answer = 0; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + answer += container_shrink_to_fit(c, type_original); + } + answer += ra_shrink_to_fit(&r->high_low_container); + return answer; +} - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); +/** + * Remove run-length encoding even when it is more space efficient + * return whether a change was applied + */ +bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + if (get_container_type(c, type_original) == RUN_CONTAINER_TYPE) { + answer = true; + if (type_original == SHARED_CONTAINER_TYPE) { + run_container_t *truec = CAST_run(CAST_shared(c)->container); + int32_t card = run_container_cardinality(truec); + container_t *c1 = convert_to_bitset_or_array_container( + truec, card, &type_after); + shared_container_free(CAST_shared(c)); // frees run as needed + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); + } else { + int32_t card = run_container_cardinality(CAST_run(c)); + container_t *c1 = convert_to_bitset_or_array_container( + CAST_run(c), card, &type_after); + run_container_free(CAST_run(c)); + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); } - - // container_t *c2_clone = container_clone(c2, type2); - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); } } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); + return answer; +} + +size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t cardinality = roaring_bitmap_get_cardinality(r); + uint64_t sizeasarray = cardinality * sizeof(uint32_t) + sizeof(uint32_t); + if (portablesize < sizeasarray) { + buf[0] = CROARING_SERIALIZATION_CONTAINER; + return roaring_bitmap_portable_serialize(r, buf + 1) + 1; + } else { + buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; + memcpy(buf + 1, &cardinality, sizeof(uint32_t)); + roaring_bitmap_to_uint32_array( + r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); + return 1 + (size_t)sizeasarray; } } -roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); +size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t sizeasarray = + roaring_bitmap_get_cardinality(r) * sizeof(uint32_t) + sizeof(uint32_t); + return portablesize < sizeasarray ? portablesize + 1 + : (size_t)sizeasarray + 1; +} + +size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r) { + return ra_portable_size_in_bytes(&r->high_low_container); +} + +roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (ans == NULL) { + return NULL; } - if (0 == length2) { - return roaring_bitmap_copy(x1); + size_t bytesread; + bool is_ok = ra_portable_deserialize(&ans->high_low_container, buf, + maxbytes, &bytesread); + if (!is_ok) { + roaring_free(ans); + return NULL; } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_xor(c1, type1, c2, type2, &result_type); + roaring_bitmap_set_copy_on_write(ans, false); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + return ans; +} - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf) { + return roaring_bitmap_portable_deserialize_safe(buf, SIZE_MAX); +} - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); +size_t roaring_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + return ra_portable_deserialize_size(buf, maxbytes); +} - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf) { + return ra_portable_serialize(&r->high_low_container, buf); +} + +roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; + + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; } - } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; + roaring_bulk_context_t context = {0}; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy(&elem, elems + i, sizeof(elem)); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; + + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize(bufaschar + 1); + } else + return (NULL); } -// inplace xor (modifies its first argument). +roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, + size_t maxbytes) { + if (maxbytes < 1) { + return NULL; + } -void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + if (maxbytes < 1 + sizeof(uint32_t)) { + return NULL; + } - if (0 == length2) return; + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; - } + // Check the buffer is big enough to contain card uint32_t elements + if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { + return NULL; + } - // XOR can have new containers inserted from x2, but can also - // lose containers when x1 and x2 are nonempty and identical. + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; + } + roaring_bulk_context_t context = {0}; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy(&elem, elems + i, sizeof(elem)); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; - // We do the computation "in place" only when c1 is not a shared container. - // Rationale: using a shared container safely with in place computation would - // require making a copy and then doing the computation in place which is likely - // less efficient than avoiding in place entirely and always generating a new - // container. + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize_safe(bufaschar + 1, + maxbytes - 1); + } else + return (NULL); +} - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_xor(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // so release - } - else { - c = container_ixor(c1, type1, c2, type2, &result_type); - } +bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, + void *ptr) { + const roaring_array_t *ra = &r->high_low_container; - if (container_nonzero_cardinality(c, result_type)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - ++pos1; - } else { - container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; - } + for (int i = 0; i < ra->size; ++i) + if (!container_iterate(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, ptr)) { + return false; + } + return true; +} - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, + uint64_t high_bits, void *ptr) { + const roaring_array_t *ra = &r->high_low_container; - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); + for (int i = 0; i < ra->size; ++i) + if (!container_iterate64(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, + high_bits, ptr)) { + return false; + } + return true; +} - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } +/**** + * begin roaring_uint32_iterator_t + *****/ - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - } +/** + * Partially initializes the iterator. Leaves it in either state: + * 1. Invalid due to `has_value = false`, or + * 2. At a container, with the high bits set, `has_value = true`. + */ +CROARING_WARN_UNUSED static bool iter_new_container_partial_init( + roaring_uint32_iterator_t *newit) { + newit->current_value = 0; + if (newit->container_index >= newit->parent->high_low_container.size || + newit->container_index < 0) { + newit->current_value = UINT32_MAX; + return (newit->has_value = false); } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); + newit->has_value = true; + // we precompute container, typecode and highbits so that successive + // iterators do not have to grab them from odd memory locations + // and have to worry about the (easily predicted) container_unwrap_shared + // call. + newit->container = + newit->parent->high_low_container.containers[newit->container_index]; + newit->typecode = + newit->parent->high_low_container.typecodes[newit->container_index]; + newit->highbits = + ((uint32_t) + newit->parent->high_low_container.keys[newit->container_index]) + << 16; + newit->container = + container_unwrap_shared(newit->container, &(newit->typecode)); + return true; +} + +/** + * Positions the iterator at the first value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadfirstvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; } + return newit->has_value; } -roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); - roaring_bitmap_set_copy_on_write(empty_bitmap, is_cow(x1) || is_cow(x2)); - return empty_bitmap; +/** + * Positions the iterator at the last value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadlastvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = container_init_iterator_last( + newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; } - if (0 == length2) { - return roaring_bitmap_copy(x1); + return newit->has_value; +} + +/** + * Positions the iterator at the smallest value that is larger than or equal to + * `val` within the current container that the iterator points at. Assumes such + * a value exists within the current container. + */ +CROARING_WARN_UNUSED static bool loadfirstvalue_largeorequal( + roaring_uint32_iterator_t *newit, uint32_t val) { + bool partial_init = iter_new_container_partial_init(newit); + assert(partial_init); + if (!partial_init) { + return false; } - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + bool found = container_iterator_lower_bound( + newit->container, newit->typecode, &newit->container_it, &value, + val & 0xFFFF); + assert(found); + if (!found) { + return false; + } + newit->current_value = newit->highbits | value; + return true; +} - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = 0; - uint16_t s2 = 0; - while (true) { - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +void roaring_iterator_init(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = 0; + newit->has_value = loadfirstvalue(newit); +} - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_andnot(c1, type1, c2, type2, - &result_type); +void roaring_iterator_init_last(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = newit->parent->high_low_container.size - 1; + newit->has_value = loadlastvalue(newit); +} - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - } else if (s1 < s2) { // s1 < s2 - const int next_pos1 = - ra_advance_until(&x1->high_low_container, s2, pos1); - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, next_pos1, - is_cow(x1)); - // TODO : perhaps some of the copy_on_write should be based on - // answer rather than x1 (more stringent?). Many similar cases - pos1 = next_pos1; - if (pos1 == length1) break; - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); - if (pos2 == length2) break; +roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + if (newit == NULL) return NULL; + roaring_iterator_init(r, newit); + return newit; +} + +roaring_uint32_iterator_t *roaring_uint32_iterator_copy( + const roaring_uint32_iterator_t *it) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + memcpy(newit, it, sizeof(roaring_uint32_iterator_t)); + return newit; +} + +bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val) { + uint16_t hb = val >> 16; + const int i = ra_get_index(&it->parent->high_low_container, hb); + if (i >= 0) { + uint32_t lowvalue = + container_maximum(it->parent->high_low_container.containers[i], + it->parent->high_low_container.typecodes[i]); + uint16_t lb = val & 0xFFFF; + if (lowvalue < lb) { + // will have to load first value of next container + it->container_index = i + 1; + } else { + // the value is necessarily within the range of the container + it->container_index = i; + it->has_value = loadfirstvalue_largeorequal(it, val); + return it->has_value; } + } else { + // there is no matching, so we are going for the next container + it->container_index = -i - 1; } - if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); + it->has_value = loadfirstvalue(it); + return it->has_value; +} + +bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it) { + if (it->container_index >= it->parent->high_low_container.size) { + return (it->has_value = false); } - return answer; + if (it->container_index < 0) { + it->container_index = 0; + return (it->has_value = loadfirstvalue(it)); + } + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_next(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); + } + it->container_index++; + return (it->has_value = loadfirstvalue(it)); } -// inplace andnot (modifies its first argument). +bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it) { + if (it->container_index < 0) { + return (it->has_value = false); + } + if (it->container_index >= it->parent->high_low_container.size) { + it->container_index = it->parent->high_low_container.size - 1; + return (it->has_value = loadlastvalue(it)); + } + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_prev(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); + } + it->container_index--; + return (it->has_value = loadlastvalue(it)); +} -void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); +uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + assert(ret == count); + return ret; + } + it->container_index++; + it->has_value = loadfirstvalue(it); + } + return ret; +} - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; - int intersection_size = 0; +void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { + roaring_free(it); +} - if (0 == length2) return; +/**** + * end of roaring_uint32_iterator_t + *****/ - if (0 == length1) { - roaring_bitmap_clear(x1); - return; +bool roaring_bitmap_equals(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; + + if (ra1->size != ra2->size) { + return false; + } + for (int i = 0; i < ra1->size; ++i) { + if (ra1->keys[i] != ra2->keys[i]) { + return false; + } + } + for (int i = 0; i < ra1->size; ++i) { + bool areequal = container_equals(ra1->containers[i], ra1->typecodes[i], + ra2->containers[i], ra2->typecodes[i]); + if (!areequal) { + return false; + } } + return true; +} - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); +bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; - // We do the computation "in place" only when c1 is not a shared container. - // Rationale: using a shared container safely with in place computation would - // require making a copy and then doing the computation in place which is likely - // less efficient than avoiding in place entirely and always generating a new - // container. + const int length1 = ra1->size, length2 = ra2->size; - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_andnot(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // release - } - else { - c = container_iandnot(c1, type1, c2, type2, &result_type); - } + int pos1 = 0, pos2 = 0; - if (container_nonzero_cardinality(c, result_type)) { - ra_replace_key_and_container_at_index(&x1->high_low_container, - intersection_size++, s1, - c, result_type); - } else { - container_free(c, result_type); - } + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = ra_get_key_at_index(ra1, (uint16_t)pos1); + const uint16_t s2 = ra_get_key_at_index(ra2, (uint16_t)pos2); + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = + ra_get_container_at_index(ra1, (uint16_t)pos1, &type1); + container_t *c2 = + ra_get_container_at_index(ra2, (uint16_t)pos2, &type2); + if (!container_is_subset(c1, type1, c2, type2)) return false; ++pos1; ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - } else if (s1 < s2) { // s1 < s2 - if (pos1 != intersection_size) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - - ra_replace_key_and_container_at_index(&x1->high_low_container, - intersection_size, s1, c1, - type1); - } - intersection_size++; - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - + return false; } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + pos2 = ra_advance_until(ra2, s1, pos2); } } + if (pos1 == length1) + return true; + else + return false; +} - if (pos1 < length1) { - // all containers between intersection_size and - // pos1 are junk. However, they have either been moved - // (thus still referenced) or involved in an iandnot - // that will clean up all containers that could not be reused. - // Thus we should not free the junk containers between - // intersection_size and pos1. - if (pos1 > intersection_size) { - // left slide of remaining items - ra_copy_range(&x1->high_low_container, pos1, length1, - intersection_size); +static void insert_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not_range(container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); + + if (container_get_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); } - // else current placement is fine - intersection_size += (length1 - pos1); + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); } - ra_downsize(&x1->high_low_container, intersection_size); } -uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; +static void inplace_flip_container(roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = container_inot_range( + container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); + // if a new container was created, the old one was already freed + if (container_get_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + } else { + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); + } - uint64_t card = 0; - for (int i = 0; i < ra->size; ++i) - card += container_get_cardinality(ra->containers[i], ra->typecodes[i]); - return card; + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); + } } -uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, - uint64_t range_start, - uint64_t range_end) { - const roaring_array_t *ra = &r->high_low_container; - - if (range_end > UINT32_MAX) { - range_end = UINT32_MAX + UINT64_C(1); - } - if (range_start >= range_end) { - return 0; +static void insert_fully_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, + uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not(container_to_flip, ctype_in, &ctype_out); + if (container_get_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); + } + } else { + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); } - range_end--; // make range_end inclusive - // now we have: 0 <= range_start <= range_end <= UINT32_MAX - - uint16_t minhb = range_start >> 16; - uint16_t maxhb = range_end >> 16; - - uint64_t card = 0; +} - int i = ra_get_index(ra, minhb); +static void inplace_fully_flip_container(roaring_array_t *x1_arr, uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; if (i >= 0) { - if (minhb == maxhb) { - card += container_rank(ra->containers[i], ra->typecodes[i], - range_end & 0xffff); + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_inot(container_to_flip, ctype_in, &ctype_out); + + if (container_get_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); } else { - card += container_get_cardinality(ra->containers[i], - ra->typecodes[i]); - } - if ((range_start & 0xffff) != 0) { - card -= container_rank(ra->containers[i], ra->typecodes[i], - (range_start & 0xffff) - 1); + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); } - i++; + } else { - i = -i - 1; + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); } +} - for (; i < ra->size; i++) { - uint16_t key = ra->keys[i]; - if (key < maxhb) { - card += container_get_cardinality(ra->containers[i], - ra->typecodes[i]); - } else if (key == maxhb) { - card += container_rank(ra->containers[i], ra->typecodes[i], - range_end & 0xffff); - break; - } else { - break; - } +roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *x1, + uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end) { + return roaring_bitmap_copy(x1); + } + if (range_end >= UINT64_C(0x100000000)) { + range_end = UINT64_C(0x100000000); } - return card; -} + roaring_bitmap_t *ans = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(ans, is_cow(x1)); + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; // & 0xFFFF; + uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); + const uint16_t lb_end = (uint16_t)(range_end - 1); // & 0xFFFF; -bool roaring_bitmap_is_empty(const roaring_bitmap_t *r) { - return r->high_low_container.size == 0; -} + ra_append_copies_until(&ans->high_low_container, &x1->high_low_container, + hb_start, is_cow(x1)); + if (hb_start == hb_end) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, + lb_start, 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } -void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans) { - ra_to_uint32_array(&r->high_low_container, ans); -} + if (lb_end != 0xFFFF) --hb_end; // later we'll handle the partial block -bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, - size_t offset, size_t limit, - uint32_t *ans) { - return ra_range_uint32_array(&r->high_low_container, offset, limit, ans); -} + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + insert_fully_flipped_container(&ans->high_low_container, + &x1->high_low_container, + (uint16_t)hb); + } -/** convert array and bitmap containers to run containers when it is more - * efficient; - * also convert from run containers when more space efficient. Returns - * true if the result has at least one run container. -*/ -bool roaring_bitmap_run_optimize(roaring_bitmap_t *r) { - bool answer = false; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original, type_after; - ra_unshare_container_at_index( - &r->high_low_container, i); // TODO: this introduces extra cloning! - container_t *c = ra_get_container_at_index(&r->high_low_container, i, - &type_original); - container_t *c1 = convert_run_optimize(c, type_original, &type_after); - if (type_after == RUN_CONTAINER_TYPE) { - answer = true; + // handle a partial final container + if (lb_end != 0xFFFF) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; } - ra_set_container_at_index(&r->high_low_container, i, c1, type_after); } - return answer; + ra_append_copies_after(&ans->high_low_container, &x1->high_low_container, + hb_end, is_cow(x1)); + return ans; } -size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r) { - size_t answer = 0; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original; - container_t *c = ra_get_container_at_index(&r->high_low_container, i, - &type_original); - answer += container_shrink_to_fit(c, type_original); +void roaring_bitmap_flip_inplace(roaring_bitmap_t *x1, uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end) { + return; // empty range + } + if (range_end >= UINT64_C(0x100000000)) { + range_end = UINT64_C(0x100000000); } - answer += ra_shrink_to_fit(&r->high_low_container); - return answer; -} -/** - * Remove run-length encoding even when it is more space efficient - * return whether a change was applied - */ -bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r) { - bool answer = false; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original, type_after; - container_t *c = ra_get_container_at_index(&r->high_low_container, i, - &type_original); - if (get_container_type(c, type_original) == RUN_CONTAINER_TYPE) { - answer = true; - if (type_original == SHARED_CONTAINER_TYPE) { - run_container_t *truec = CAST_run(CAST_shared(c)->container); - int32_t card = run_container_cardinality(truec); - container_t *c1 = convert_to_bitset_or_array_container( - truec, card, &type_after); - shared_container_free(CAST_shared(c)); // frees run as needed - ra_set_container_at_index(&r->high_low_container, i, c1, - type_after); + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; + uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); + const uint16_t lb_end = (uint16_t)(range_end - 1); - } else { - int32_t card = run_container_cardinality(CAST_run(c)); - container_t *c1 = convert_to_bitset_or_array_container( - CAST_run(c), card, &type_after); - run_container_free(CAST_run(c)); - ra_set_container_at_index(&r->high_low_container, i, c1, - type_after); - } + if (hb_start == hb_end) { + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } + + if (lb_end != 0xFFFF) --hb_end; + + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + inplace_fully_flip_container(&x1->high_low_container, (uint16_t)hb); + } + // handle a partial final container + if (lb_end != 0xFFFF) { + inplace_flip_container(&x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; } } - return answer; } -size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { - size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); - uint64_t cardinality = roaring_bitmap_get_cardinality(r); - uint64_t sizeasarray = cardinality * sizeof(uint32_t) + sizeof(uint32_t); - if (portablesize < sizeasarray) { - buf[0] = CROARING_SERIALIZATION_CONTAINER; - return roaring_bitmap_portable_serialize(r, buf + 1) + 1; - } else { - buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; - memcpy(buf + 1, &cardinality, sizeof(uint32_t)); - roaring_bitmap_to_uint32_array( - r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); - return 1 + (size_t)sizeasarray; +static void offset_append_with_merge(roaring_array_t *ra, int k, container_t *c, + uint8_t t) { + int size = ra_get_size(ra); + if (size == 0 || ra_get_key_at_index(ra, (uint16_t)(size - 1)) != k) { + // No merge. + ra_append(ra, (uint16_t)k, c, t); + return; } -} -size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r) { - size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); - uint64_t sizeasarray = roaring_bitmap_get_cardinality(r) * sizeof(uint32_t) + - sizeof(uint32_t); - return portablesize < sizeasarray ? portablesize + 1 : (size_t)sizeasarray + 1; -} + uint8_t last_t, new_t; + container_t *last_c, *new_c; -size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r) { - return ra_portable_size_in_bytes(&r->high_low_container); -} + // NOTE: we don't need to unwrap here, since we added last_c ourselves + // we have the certainty it's not a shared container. + // The same applies to c, as it's the result of calling container_offset. + last_c = ra_get_container_at_index(ra, (uint16_t)(size - 1), &last_t); + new_c = container_ior(last_c, last_t, c, t, &new_t); + ra_set_container_at_index(ra, size - 1, new_c, new_t); -roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, size_t maxbytes) { - roaring_bitmap_t *ans = - (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); - if (ans == NULL) { - return NULL; - } - size_t bytesread; - bool is_ok = ra_portable_deserialize(&ans->high_low_container, buf, maxbytes, &bytesread); - if (!is_ok) { - roaring_free(ans); - return NULL; - } - roaring_bitmap_set_copy_on_write(ans, false); - if (!is_ok) { - roaring_free(ans); - return NULL; + // Comparison of pointers of different origin is UB (or so claim some + // compiler makers), so we compare their bit representation only. + if ((uintptr_t)last_c != (uintptr_t)new_c) { + container_free(last_c, last_t); } - return ans; -} - -roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf) { - return roaring_bitmap_portable_deserialize_safe(buf, SIZE_MAX); + container_free(c, t); } +// roaring_bitmap_add_offset adds the value 'offset' to each and every value in +// a bitmap, generating a new bitmap in the process. If offset + element is +// outside of the range [0,2^32), that the element will be dropped. +// We need "offset" to be 64 bits because we want to support values +// between -0xFFFFFFFF up to +0xFFFFFFFF. +roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, + int64_t offset) { + roaring_bitmap_t *answer; + roaring_array_t *ans_ra; + int64_t container_offset; + uint16_t in_offset; -size_t roaring_bitmap_portable_deserialize_size(const char *buf, size_t maxbytes) { - return ra_portable_deserialize_size(buf, maxbytes); -} + const roaring_array_t *bm_ra = &bm->high_low_container; + int length = bm_ra->size; + if (offset == 0) { + return roaring_bitmap_copy(bm); + } -size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, - char *buf) { - return ra_portable_serialize(&r->high_low_container, buf); -} + container_offset = offset >> 16; + in_offset = (uint16_t)(offset - container_offset * (1 << 16)); -roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { - const char *bufaschar = (const char *)buf; - if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { - /* This looks like a compressed set of uint32_t elements */ - uint32_t card; + answer = roaring_bitmap_create(); + bool cow = is_cow(bm); + roaring_bitmap_set_copy_on_write(answer, cow); - memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + ans_ra = &answer->high_low_container; - const uint32_t *elems = - (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); - - roaring_bitmap_t *bitmap = roaring_bitmap_create(); - if (bitmap == NULL) { - return NULL; - } - roaring_bulk_context_t context = {0}; - for (uint32_t i = 0; i < card; i++) { - // elems may not be aligned, read with memcpy - uint32_t elem; - memcpy(&elem, elems + i, sizeof(elem)); - roaring_bitmap_add_bulk(bitmap, &context, elem); - } - return bitmap; + if (in_offset == 0) { + ans_ra = &answer->high_low_container; - } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { - return roaring_bitmap_portable_deserialize(bufaschar + 1); - } else - return (NULL); -} + for (int i = 0, j = 0; i < length; ++i) { + int64_t key = ra_get_key_at_index(bm_ra, (uint16_t)i); + key += container_offset; -roaring_bitmap_t* roaring_bitmap_deserialize_safe(const void *buf, size_t maxbytes) { - if (maxbytes < 1) { - return NULL; + if (key < 0 || key >= (1 << 16)) { + continue; + } + ra_append_copy(ans_ra, bm_ra, (uint16_t)i, cow); + ans_ra->keys[j++] = (uint16_t)key; + } + return answer; } - const char *bufaschar = (const char *)buf; - if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { - if (maxbytes < 1 + sizeof(uint32_t)) { - return NULL; - } + uint8_t t; + const container_t *c; + container_t *lo, *hi, **lo_ptr, **hi_ptr; + int64_t k; - /* This looks like a compressed set of uint32_t elements */ - uint32_t card; - memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + for (int i = 0; i < length; ++i) { + lo = hi = NULL; + lo_ptr = hi_ptr = NULL; - // Check the buffer is big enough to contain card uint32_t elements - if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { - return NULL; + k = ra_get_key_at_index(bm_ra, (uint16_t)i) + container_offset; + if (k >= 0 && k < (1 << 16)) { + lo_ptr = &lo; + } + if (k + 1 >= 0 && k + 1 < (1 << 16)) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + continue; } + c = ra_get_container_at_index(bm_ra, (uint16_t)i, &t); + c = container_unwrap_shared(c, &t); - const uint32_t *elems = - (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); - - roaring_bitmap_t *bitmap = roaring_bitmap_create(); - if (bitmap == NULL) { - return NULL; + container_add_offset(c, t, lo_ptr, hi_ptr, in_offset); + if (lo != NULL) { + offset_append_with_merge(ans_ra, (int)k, lo, t); } - roaring_bulk_context_t context = {0}; - for (uint32_t i = 0; i < card; i++) { - // elems may not be aligned, read with memcpy - uint32_t elem; - memcpy(&elem, elems + i, sizeof(elem)); - roaring_bitmap_add_bulk(bitmap, &context, elem); + if (hi != NULL) { + ra_append(ans_ra, (uint16_t)(k + 1), hi, t); } - return bitmap; - - } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { - return roaring_bitmap_portable_deserialize_safe(bufaschar + 1, maxbytes - 1); - } else - return (NULL); + // the `lo` and `hi` container type always keep same as container `c`. + // in the case of `container_add_offset` on bitset container, `lo` and + // `hi` may has small cardinality, they must be repaired to array + // container. + } + + roaring_bitmap_repair_after_lazy(answer); // do required type conversions. + return answer; } -bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, - void *ptr) { - const roaring_array_t *ra = &r->high_low_container; +roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c; + if (bitsetconversion && + (get_container_type(c1, type1) != BITSET_CONTAINER_TYPE) && + (get_container_type(c2, type2) != BITSET_CONTAINER_TYPE)) { + container_t *newc1 = + container_mutable_unwrap_shared(c1, &type1); + newc1 = container_to_bitset(newc1, type1); + type1 = BITSET_CONTAINER_TYPE; + c = container_lazy_ior(newc1, type1, c2, type2, &result_type); + if (c != newc1) { // should not happen + container_free(newc1, type1); + } + } else { + c = container_lazy_or(c1, type1, c2, type2, &result_type); + } + // since we assume that the initial containers are non-empty, + // the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - for (int i = 0; i < ra->size; ++i) - if (!container_iterate(ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16, - iterator, ptr)) { - return false; + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - return true; + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; } -bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, - uint64_t high_bits, void *ptr) { - const roaring_array_t *ra = &r->high_low_container; +void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; - for (int i = 0; i < ra->size; ++i) - if (!container_iterate64( - ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16, iterator, - high_bits, ptr)) { - return false; - } - return true; -} + if (0 == length2) return; -/**** -* begin roaring_uint32_iterator_t -*****/ - -// Partially initializes the roaring iterator when it begins looking at -// a new container. -static bool iter_new_container_partial_init(roaring_uint32_iterator_t *newit) { - newit->in_container_index = 0; - newit->run_index = 0; - newit->current_value = 0; - if (newit->container_index >= newit->parent->high_low_container.size || - newit->container_index < 0) { - newit->current_value = UINT32_MAX; - return (newit->has_value = false); + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; } - // assume not empty - newit->has_value = true; - // we precompute container, typecode and highbits so that successive - // iterators do not have to grab them from odd memory locations - // and have to worry about the (easily predicted) container_unwrap_shared - // call. - newit->container = - newit->parent->high_low_container.containers[newit->container_index]; - newit->typecode = - newit->parent->high_low_container.typecodes[newit->container_index]; - newit->highbits = - ((uint32_t) - newit->parent->high_low_container.keys[newit->container_index]) - << 16; - newit->container = - container_unwrap_shared(newit->container, &(newit->typecode)); - return newit->has_value; -} + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + if ((bitsetconversion == false) || + (get_container_type(c1, type1) == BITSET_CONTAINER_TYPE)) { + c1 = get_writable_copy_if_shared(c1, &type1); + } else { + // convert to bitset + container_t *old_c1 = c1; + uint8_t old_type1 = type1; + c1 = container_mutable_unwrap_shared(c1, &type1); + c1 = container_to_bitset(c1, type1); + container_free(old_c1, old_type1); + type1 = BITSET_CONTAINER_TYPE; + } -static bool loadfirstvalue(roaring_uint32_iterator_t *newit) { - if (!iter_new_container_partial_init(newit)) - return newit->has_value; + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + container_lazy_ior(c1, type1, c2, type2, &result_type); - switch (newit->typecode) { - case BITSET_CONTAINER_TYPE: { - const bitset_container_t *bc = const_CAST_bitset(newit->container); + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } - uint32_t wordindex = 0; - uint64_t word; - while ((word = bc->words[wordindex]) == 0) { - wordindex++; // advance + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); } - // here "word" is non-zero - newit->in_container_index = wordindex * 64 + roaring_trailing_zeroes(word); - newit->current_value = newit->highbits | newit->in_container_index; - break; } - - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = const_CAST_array(newit->container); - newit->current_value = newit->highbits | ac->array[0]; - break; } - - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(newit->container); - newit->current_value = newit->highbits | rc->runs[0].value; - break; } - - default: - // if this ever happens, bug! - assert(false); - } // switch (typecode) - return true; -} - -static bool loadlastvalue(roaring_uint32_iterator_t* newit) { - if (!iter_new_container_partial_init(newit)) - return newit->has_value; + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - switch(newit->typecode) { - case BITSET_CONTAINER_TYPE: { - uint32_t wordindex = BITSET_CONTAINER_SIZE_IN_WORDS - 1; - uint64_t word; - const bitset_container_t* bitset_container = (const bitset_container_t*)newit->container; - while ((word = bitset_container->words[wordindex]) == 0) - --wordindex; + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - int num_leading_zeros = roaring_leading_zeroes(word); - newit->in_container_index = (wordindex * 64) + (63 - num_leading_zeros); - newit->current_value = newit->highbits | newit->in_container_index; - break; - } - case ARRAY_CONTAINER_TYPE: { - const array_container_t* array_container = (const array_container_t*)newit->container; - newit->in_container_index = array_container->cardinality - 1; - newit->current_value = newit->highbits | array_container->array[newit->in_container_index]; - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t* run_container = (const run_container_t*)newit->container; - newit->run_index = run_container->n_runs - 1; - const rle16_t* last_run = &run_container->runs[newit->run_index]; - newit->current_value = newit->highbits | (last_run->value + last_run->length); - break; + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - default: - // if this ever happens, bug! - assert(false); } - return true; + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } } -// prerequesite: the value should be in range of the container -static bool loadfirstvalue_largeorequal(roaring_uint32_iterator_t *newit, uint32_t val) { - // Don't have to check return value because of prerequisite - iter_new_container_partial_init(newit); - uint16_t lb = val & 0xFFFF; - - switch (newit->typecode) { - case BITSET_CONTAINER_TYPE: { - const bitset_container_t *bc = const_CAST_bitset(newit->container); - newit->in_container_index = - bitset_container_index_equalorlarger(bc, lb); - newit->current_value = newit->highbits | newit->in_container_index; - break; } - - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = const_CAST_array(newit->container); - newit->in_container_index = - array_container_index_equalorlarger(ac, lb); - newit->current_value = - newit->highbits | ac->array[newit->in_container_index]; - break; } +roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_lazy_xor(c1, type1, c2, type2, &result_type); - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(newit->container); - newit->run_index = run_container_index_equalorlarger(rc, lb); - if (rc->runs[newit->run_index].value <= lb) { - newit->current_value = val; + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); } else { - newit->current_value = - newit->highbits | rc->runs[newit->run_index].value; + container_free(c, result_type); } - break; } - - default: - roaring_unreachable; - } - - return true; -} - -void roaring_init_iterator(const roaring_bitmap_t *r, - roaring_uint32_iterator_t *newit) { - newit->parent = r; - newit->container_index = 0; - newit->has_value = loadfirstvalue(newit); -} - -void roaring_init_iterator_last(const roaring_bitmap_t *r, - roaring_uint32_iterator_t *newit) { - newit->parent = r; - newit->container_index = newit->parent->high_low_container.size - 1; - newit->has_value = loadlastvalue(newit); -} -roaring_uint32_iterator_t *roaring_create_iterator(const roaring_bitmap_t *r) { - roaring_uint32_iterator_t *newit = - (roaring_uint32_iterator_t *)roaring_malloc(sizeof(roaring_uint32_iterator_t)); - if (newit == NULL) return NULL; - roaring_init_iterator(r, newit); - return newit; -} + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); -roaring_uint32_iterator_t *roaring_copy_uint32_iterator( - const roaring_uint32_iterator_t *it) { - roaring_uint32_iterator_t *newit = - (roaring_uint32_iterator_t *)roaring_malloc(sizeof(roaring_uint32_iterator_t)); - memcpy(newit, it, sizeof(roaring_uint32_iterator_t)); - return newit; -} + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); -bool roaring_move_uint32_iterator_equalorlarger(roaring_uint32_iterator_t *it, uint32_t val) { - uint16_t hb = val >> 16; - const int i = ra_get_index(& it->parent->high_low_container, hb); - if (i >= 0) { - uint32_t lowvalue = container_maximum(it->parent->high_low_container.containers[i], it->parent->high_low_container.typecodes[i]); - uint16_t lb = val & 0xFFFF; - if(lowvalue < lb ) { - it->container_index = i+1; // will have to load first value of next container - } else {// the value is necessarily within the range of the container - it->container_index = i; - it->has_value = loadfirstvalue_largeorequal(it, val); - return it->has_value; - } - } else { - // there is no matching, so we are going for the next container - it->container_index = -i-1; + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); } - it->has_value = loadfirstvalue(it); - return it->has_value; + return answer; } +void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; -bool roaring_advance_uint32_iterator(roaring_uint32_iterator_t *it) { - if (it->container_index >= it->parent->high_low_container.size) { - return (it->has_value = false); - } - if (it->container_index < 0) { - it->container_index = 0; - return (it->has_value = loadfirstvalue(it)); - } - - switch (it->typecode) { - case BITSET_CONTAINER_TYPE: { - const bitset_container_t *bc = const_CAST_bitset(it->container); - it->in_container_index++; + if (0 == length2) return; - uint32_t wordindex = it->in_container_index / 64; - if (wordindex >= BITSET_CONTAINER_SIZE_IN_WORDS) break; + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); - uint64_t word = bc->words[wordindex] & - (UINT64_MAX << (it->in_container_index % 64)); - // next part could be optimized/simplified - while ((word == 0) && - (wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS)) { - wordindex++; - word = bc->words[wordindex]; - } - if (word != 0) { - it->in_container_index = wordindex * 64 + roaring_trailing_zeroes(word); - it->current_value = it->highbits | it->in_container_index; - return (it->has_value = true); - } - break; } + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = const_CAST_array(it->container); - it->in_container_index++; - if (it->in_container_index < ac->cardinality) { - it->current_value = - it->highbits | ac->array[it->in_container_index]; - return (it->has_value = true); + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_lazy_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_lazy_ixor(c1, type1, c2, type2, &result_type); } - break; } - case RUN_CONTAINER_TYPE: { - if(it->current_value == UINT32_MAX) { // avoid overflow to zero - return (it->has_value = false); + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + ra_remove_at_index(&x1->high_low_container, pos1); + --length1; } + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - const run_container_t* rc = const_CAST_run(it->container); - uint32_t limit = (it->highbits | (rc->runs[it->run_index].value + - rc->runs[it->run_index].length)); - if (++it->current_value <= limit) { - return (it->has_value = true); - } + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - if (++it->run_index < rc->n_runs) { // Assume the run has a value - it->current_value = - it->highbits | rc->runs[it->run_index].value; - return (it->has_value = true); + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); } - break; + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - - default: - roaring_unreachable; - } - - // moving to next container - it->container_index++; - return (it->has_value = loadfirstvalue(it)); -} - -bool roaring_previous_uint32_iterator(roaring_uint32_iterator_t *it) { - if (it->container_index < 0) { - return (it->has_value = false); } - if (it->container_index >= it->parent->high_low_container.size) { - it->container_index = it->parent->high_low_container.size - 1; - return (it->has_value = loadlastvalue(it)); + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); } +} - switch (it->typecode) { - case BITSET_CONTAINER_TYPE: { - if (--it->in_container_index < 0) - break; - - const bitset_container_t* bitset_container = (const bitset_container_t*)it->container; - int32_t wordindex = it->in_container_index / 64; - uint64_t word = bitset_container->words[wordindex] & (UINT64_MAX >> (63 - (it->in_container_index % 64))); +void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r) { + roaring_array_t *ra = &r->high_low_container; - while (word == 0 && --wordindex >= 0) { - word = bitset_container->words[wordindex]; - } - if (word == 0) - break; + for (int i = 0; i < ra->size; ++i) { + const uint8_t old_type = ra->typecodes[i]; + container_t *old_c = ra->containers[i]; + uint8_t new_type = old_type; + container_t *new_c = container_repair_after_lazy(old_c, &new_type); + ra->containers[i] = new_c; + ra->typecodes[i] = new_type; + } +} - int num_leading_zeros = roaring_leading_zeroes(word); - it->in_container_index = (wordindex * 64) + (63 - num_leading_zeros); - it->current_value = it->highbits | it->in_container_index; - return (it->has_value = true); +/** + * roaring_bitmap_rank returns the number of integers that are smaller or equal + * to x. + */ +uint64_t roaring_bitmap_rank(const roaring_bitmap_t *bm, uint32_t x) { + uint64_t size = 0; + uint32_t xhigh = x >> 16; + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + return size + container_rank(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], + x & 0xFFFF); + } else { + return size; } - case ARRAY_CONTAINER_TYPE: { - if (--it->in_container_index < 0) - break; + } + return size; +} +void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + uint64_t size = 0; - const array_container_t* array_container = (const array_container_t*)it->container; - it->current_value = it->highbits | array_container->array[it->in_container_index]; - return (it->has_value = true); + int i = 0; + const uint32_t *iter = begin; + while (i < bm->high_low_container.size && iter != end) { + uint32_t x = *iter; + uint32_t xhigh = x >> 16; + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + i++; + } else if (xhigh == key) { + uint32_t consumed = container_rank_many( + bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], size, iter, end, ans); + iter += consumed; + ans += consumed; + } else { + *(ans++) = size; + iter++; } - case RUN_CONTAINER_TYPE: { - if(it->current_value == 0) - return (it->has_value = false); - - const run_container_t* run_container = (const run_container_t*)it->container; - if (--it->current_value >= (it->highbits | run_container->runs[it->run_index].value)) { - return (it->has_value = true); - } + } +} - if (--it->run_index < 0) - break; +/** + * roaring_bitmap_get_index returns the index of x, if not exsist return -1. + */ +int64_t roaring_bitmap_get_index(const roaring_bitmap_t *bm, uint32_t x) { + int64_t index = 0; + const uint16_t xhigh = x >> 16; + int32_t high_idx = ra_get_index(&bm->high_low_container, xhigh); + if (high_idx < 0) return -1; - it->current_value = it->highbits | (run_container->runs[it->run_index].value + - run_container->runs[it->run_index].length); - return (it->has_value = true); + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + index += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + int32_t low_idx = container_get_index( + bm->high_low_container.containers[high_idx], + bm->high_low_container.typecodes[high_idx], x & 0xFFFF); + if (low_idx < 0) return -1; + return index + low_idx; + } else { + return -1; } - default: - // if this ever happens, bug! - assert(false); - } // switch (typecode) - - // moving to previous container - it->container_index--; - return (it->has_value = loadlastvalue(it)); + } + return index; } -uint32_t roaring_read_uint32_iterator(roaring_uint32_iterator_t *it, uint32_t* buf, uint32_t count) { - uint32_t ret = 0; - uint32_t num_values; - uint32_t wordindex; // used for bitsets - uint64_t word; // used for bitsets - const array_container_t* acont; //TODO remove - const run_container_t* rcont; //TODO remove - const bitset_container_t* bcont; //TODO remove - - while (it->has_value && ret < count) { - switch (it->typecode) { - case BITSET_CONTAINER_TYPE: - bcont = const_CAST_bitset(it->container); - wordindex = it->in_container_index / 64; - word = bcont->words[wordindex] & (UINT64_MAX << (it->in_container_index % 64)); - do { - while (word != 0 && ret < count) { - buf[0] = it->highbits | (wordindex * 64 + roaring_trailing_zeroes(word)); - word = word & (word - 1); - buf++; - ret++; - } - while (word == 0 && wordindex+1 < BITSET_CONTAINER_SIZE_IN_WORDS) { - wordindex++; - word = bcont->words[wordindex]; - } - } while (word != 0 && ret < count); - it->has_value = (word != 0); - if (it->has_value) { - it->in_container_index = wordindex * 64 + roaring_trailing_zeroes(word); - it->current_value = it->highbits | it->in_container_index; - } - break; - case ARRAY_CONTAINER_TYPE: - acont = const_CAST_array(it->container); - num_values = minimum_uint32(acont->cardinality - it->in_container_index, count - ret); - for (uint32_t i = 0; i < num_values; i++) { - buf[i] = it->highbits | acont->array[it->in_container_index + i]; - } - buf += num_values; - ret += num_values; - it->in_container_index += num_values; - it->has_value = (it->in_container_index < acont->cardinality); - if (it->has_value) { - it->current_value = it->highbits | acont->array[it->in_container_index]; - } - break; - case RUN_CONTAINER_TYPE: - rcont = const_CAST_run(it->container); - //"in_run_index" name is misleading, read it as "max_value_in_current_run" - do { - uint32_t largest_run_value = it->highbits | (rcont->runs[it->run_index].value + rcont->runs[it->run_index].length); - num_values = minimum_uint32(largest_run_value - it->current_value + 1, count - ret); - for (uint32_t i = 0; i < num_values; i++) { - buf[i] = it->current_value + i; - } - it->current_value += num_values; // this can overflow to zero: UINT32_MAX+1=0 - buf += num_values; - ret += num_values; - - if (it->current_value > largest_run_value || it->current_value == 0) { - it->run_index++; - if (it->run_index < rcont->n_runs) { - it->current_value = it->highbits | rcont->runs[it->run_index].value; - } else { - it->has_value = false; - } - } - } while ((ret < count) && it->has_value); - break; - default: - assert(false); - } - if (it->has_value) { - assert(ret == count); - return ret; +/** + * roaring_bitmap_smallest returns the smallest value in the set. + * Returns UINT32_MAX if the set is empty. + */ +uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *c = bm->high_low_container.containers[0]; + uint8_t type = bm->high_low_container.typecodes[0]; + uint32_t key = bm->high_low_container.keys[0]; + uint32_t lowvalue = container_minimum(c, type); + return lowvalue | (key << 16); } - it->container_index++; - it->has_value = loadfirstvalue(it); - } - return ret; + return UINT32_MAX; } +/** + * roaring_bitmap_smallest returns the greatest value in the set. + * Returns 0 if the set is empty. + */ +uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *container = + bm->high_low_container.containers[bm->high_low_container.size - 1]; + uint8_t typecode = + bm->high_low_container.typecodes[bm->high_low_container.size - 1]; + uint32_t key = + bm->high_low_container.keys[bm->high_low_container.size - 1]; + uint32_t lowvalue = container_maximum(container, typecode); + return lowvalue | (key << 16); + } + return 0; +} +bool roaring_bitmap_select(const roaring_bitmap_t *bm, uint32_t rank, + uint32_t *element) { + container_t *container; + uint8_t typecode; + uint16_t key; + uint32_t start_rank = 0; + int i = 0; + bool valid = false; + while (!valid && i < bm->high_low_container.size) { + container = bm->high_low_container.containers[i]; + typecode = bm->high_low_container.typecodes[i]; + valid = + container_select(container, typecode, &start_rank, rank, element); + i++; + } -void roaring_free_uint32_iterator(roaring_uint32_iterator_t *it) { roaring_free(it); } - -/**** -* end of roaring_uint32_iterator_t -*****/ - -bool roaring_bitmap_equals(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - const roaring_array_t *ra1 = &r1->high_low_container; - const roaring_array_t *ra2 = &r2->high_low_container; - - if (ra1->size != ra2->size) { + if (valid) { + key = bm->high_low_container.keys[i - 1]; + *element |= (((uint32_t)key) << 16); // w/o cast, key promotes signed + return true; + } else return false; - } - for (int i = 0; i < ra1->size; ++i) { - if (ra1->keys[i] != ra2->keys[i]) { - return false; - } - } - for (int i = 0; i < ra1->size; ++i) { - bool areequal = container_equals(ra1->containers[i], - ra1->typecodes[i], - ra2->containers[i], - ra2->typecodes[i]); - if (!areequal) { - return false; - } - } - return true; } -bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - const roaring_array_t *ra1 = &r1->high_low_container; - const roaring_array_t *ra2 = &r2->high_low_container; - - const int length1 = ra1->size, - length2 = ra2->size; - +bool roaring_bitmap_intersect(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; int pos1 = 0, pos2 = 0; while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(ra1, pos1); - const uint16_t s2 = ra_get_key_at_index(ra2, pos2); + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); if (s1 == s2) { uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index(ra1, pos1, &type1); - container_t *c2 = ra_get_container_at_index(ra2, pos2, &type2); - if (!container_is_subset(c1, type1, c2, type2)) - return false; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + if (container_intersect(c1, type1, c2, type2)) return true; ++pos1; ++pos2; } else if (s1 < s2) { // s1 < s2 - return false; + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); } else { // s1 > s2 - pos2 = ra_advance_until(ra2, s1, pos2); + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } } - if (pos1 == length1) - return true; - else + return answer != 0; +} + +bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, + uint64_t y) { + if (x >= y) { + // Empty range. + return false; + } + roaring_uint32_iterator_t it; + roaring_iterator_init(bm, &it); + if (!roaring_uint32_iterator_move_equalorlarger(&it, (uint32_t)x)) { + // No values above x. + return false; + } + if (it.current_value >= y) { + // No values below y. return false; + } + return true; } -static void insert_flipped_container(roaring_array_t *ans_arr, - const roaring_array_t *x1_arr, uint16_t hb, - uint16_t lb_start, uint16_t lb_end) { - const int i = ra_get_index(x1_arr, hb); - const int j = ra_get_index(ans_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, i, &ctype_in); - flipped_container = - container_not_range(container_to_flip, ctype_in, (uint32_t)lb_start, - (uint32_t)(lb_end + 1), &ctype_out); +uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; + int pos1 = 0, pos2 = 0; + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - if (container_get_cardinality(flipped_container, ctype_out)) - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); - else { - container_free(flipped_container, ctype_out); + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + answer += container_and_cardinality(c1, type1, c2, type2); + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } - } else { - flipped_container = container_range_of_ones( - (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); } + return answer; } -static void inplace_flip_container(roaring_array_t *x1_arr, uint16_t hb, - uint16_t lb_start, uint16_t lb_end) { - const int i = ra_get_index(x1_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, i, &ctype_in); - flipped_container = container_inot_range( - container_to_flip, ctype_in, (uint32_t)lb_start, - (uint32_t)(lb_end + 1), &ctype_out); - // if a new container was created, the old one was already freed - if (container_get_cardinality(flipped_container, ctype_out)) { - ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); - } else { - container_free(flipped_container, ctype_out); - ra_remove_at_index(x1_arr, i); - } +double roaring_bitmap_jaccard_index(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return (double)inter / (double)(c1 + c2 - inter); +} - } else { - flipped_container = container_range_of_ones( - (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); - ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, - ctype_out); - } +uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - inter; } -static void insert_fully_flipped_container(roaring_array_t *ans_arr, - const roaring_array_t *x1_arr, - uint16_t hb) { - const int i = ra_get_index(x1_arr, hb); - const int j = ra_get_index(ans_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, i, &ctype_in); - flipped_container = - container_not(container_to_flip, ctype_in, &ctype_out); - if (container_get_cardinality(flipped_container, ctype_out)) - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); - else { - container_free(flipped_container, ctype_out); - } - } else { - flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); - } +uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 - inter; } -static void inplace_fully_flip_container(roaring_array_t *x1_arr, uint16_t hb) { - const int i = ra_get_index(x1_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, i, &ctype_in); - flipped_container = - container_inot(container_to_flip, ctype_in, &ctype_out); +uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - 2 * inter; +} - if (container_get_cardinality(flipped_container, ctype_out)) { - ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); - } else { - container_free(flipped_container, ctype_out); - ra_remove_at_index(x1_arr, i); - } +bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + /* + * the next function call involves a binary search and lots of branching. + */ + int32_t i = ra_get_index(&r->high_low_container, hb); + if (i < 0) return false; - } else { - flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); - ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, - ctype_out); - } + uint8_t typecode; + // next call ought to be cheap + container_t *container = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &typecode); + // rest might be a tad expensive, possibly involving another round of binary + // search + return container_contains(container, val & 0xFFFF, typecode); } -roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *x1, - uint64_t range_start, - uint64_t range_end) { - if (range_start >= range_end) { - return roaring_bitmap_copy(x1); - } - if(range_end >= UINT64_C(0x100000000)) { +/** + * Check whether a range of values from range_start (included) to range_end + * (excluded) is present + */ +bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, + uint64_t range_start, uint64_t range_end) { + if (range_end >= UINT64_C(0x100000000)) { range_end = UINT64_C(0x100000000); } - - roaring_bitmap_t *ans = roaring_bitmap_create(); - roaring_bitmap_set_copy_on_write(ans, is_cow(x1)); - - uint16_t hb_start = (uint16_t)(range_start >> 16); - const uint16_t lb_start = (uint16_t)range_start; // & 0xFFFF; - uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); - const uint16_t lb_end = (uint16_t)(range_end - 1); // & 0xFFFF; - - ra_append_copies_until(&ans->high_low_container, &x1->high_low_container, - hb_start, is_cow(x1)); - if (hb_start == hb_end) { - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_start, lb_start, - lb_end); - } else { - // start and end containers are distinct - if (lb_start > 0) { - // handle first (partial) container - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_start, - lb_start, 0xFFFF); - ++hb_start; // for the full containers. Can't wrap. + if (range_start >= range_end) + return true; // empty range are always contained! + if (range_end - range_start == 1) + return roaring_bitmap_contains(r, (uint32_t)range_start); + uint16_t hb_rs = (uint16_t)(range_start >> 16); + uint16_t hb_re = (uint16_t)((range_end - 1) >> 16); + const int32_t span = hb_re - hb_rs; + const int32_t hlc_sz = ra_get_size(&r->high_low_container); + if (hlc_sz < span + 1) { + return false; + } + int32_t is = ra_get_index(&r->high_low_container, hb_rs); + int32_t ie = ra_get_index(&r->high_low_container, hb_re); + if ((ie < 0) || (is < 0) || ((ie - is) != span) || ie >= hlc_sz) { + return false; + } + const uint32_t lb_rs = range_start & 0xFFFF; + const uint32_t lb_re = ((range_end - 1) & 0xFFFF) + 1; + uint8_t type; + container_t *c = + ra_get_container_at_index(&r->high_low_container, (uint16_t)is, &type); + if (hb_rs == hb_re) { + return container_contains_range(c, lb_rs, lb_re, type); + } + if (!container_contains_range(c, lb_rs, 1 << 16, type)) { + return false; + } + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)ie, &type); + if (!container_contains_range(c, 0, lb_re, type)) { + return false; + } + for (int32_t i = is + 1; i < ie; ++i) { + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)i, + &type); + if (!container_is_full(c, type)) { + return false; } + } + return true; +} - if (lb_end != 0xFFFF) --hb_end; // later we'll handle the partial block +bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + return (roaring_bitmap_get_cardinality(r2) > + roaring_bitmap_get_cardinality(r1) && + roaring_bitmap_is_subset(r1, r2)); +} - for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { - insert_fully_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb); - } +/* + * FROZEN SERIALIZATION FORMAT DESCRIPTION + * + * -- (beginning must be aligned by 32 bytes) -- + * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * + * num_bitset_containers] rle16_t[total number of rle elements in + * all run containers] uint16_t[total number of array elements in + * all array containers] uint16_t[num_containers] + * uint16_t[num_containers] uint8_t[num_containers]
+ * uint32_t + * + *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + * and the number of containers (17 bits). + * + * stores number of elements for every container. + * Its meaning depends on container type. + * For array and bitset containers, this value is the container cardinality + * minus one. For run container, it is the number of rle_t elements (n_runs). + * + * ,, are flat arrays of elements of + * all containers of respective type. + * + * <*_data> and are kept close together because they are not accessed + * during deserilization. This may reduce IO in case of large mmaped bitmaps. + * All members have their native alignments during deserilization except + *
, which is not guaranteed to be aligned by 4 bytes. + */ - // handle a partial final container - if (lb_end != 0xFFFF) { - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_end + 1, 0, - lb_end); - ++hb_end; +size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *rb) { + const roaring_array_t *ra = &rb->high_low_container; + size_t num_bytes = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + num_bytes += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + num_bytes += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + num_bytes += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; } } - ra_append_copies_after(&ans->high_low_container, &x1->high_low_container, - hb_end, is_cow(x1)); - return ans; + num_bytes += (2 + 2 + 1) * ra->size; // keys, counts, typecodes + num_bytes += 4; // header + return num_bytes; } -void roaring_bitmap_flip_inplace(roaring_bitmap_t *x1, uint64_t range_start, - uint64_t range_end) { - if (range_start >= range_end) { - return; // empty range - } - if(range_end >= UINT64_C(0x100000000)) { - range_end = UINT64_C(0x100000000); - } +inline static void *arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; +} - uint16_t hb_start = (uint16_t)(range_start >> 16); - const uint16_t lb_start = (uint16_t)range_start; - uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); - const uint16_t lb_end = (uint16_t)(range_end - 1); +void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *rb, char *buf) { + /* + * Note: we do not require user to supply a specifically aligned buffer. + * Thus we have to use memcpy() everywhere. + */ - if (hb_start == hb_end) { - inplace_flip_container(&x1->high_low_container, hb_start, lb_start, - lb_end); - } else { - // start and end containers are distinct - if (lb_start > 0) { - // handle first (partial) container - inplace_flip_container(&x1->high_low_container, hb_start, lb_start, - 0xFFFF); - ++hb_start; // for the full containers. Can't wrap. + const roaring_array_t *ra = &rb->high_low_container; + + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + run_zone_size += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + array_zone_size += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; } + } - if (lb_end != 0xFFFF) --hb_end; + uint64_t *bitset_zone = (uint64_t *)arena_alloc(&buf, bitset_zone_size); + rle16_t *run_zone = (rle16_t *)arena_alloc(&buf, run_zone_size); + uint16_t *array_zone = (uint16_t *)arena_alloc(&buf, array_zone_size); + uint16_t *key_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint16_t *count_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint8_t *typecode_zone = (uint8_t *)arena_alloc(&buf, ra->size); + uint32_t *header_zone = (uint32_t *)arena_alloc(&buf, 4); - for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { - inplace_fully_flip_container(&x1->high_low_container, hb); - } - // handle a partial final container - if (lb_end != 0xFFFF) { - inplace_flip_container(&x1->high_low_container, hb_end + 1, 0, - lb_end); - ++hb_end; + for (int32_t i = 0; i < ra->size; i++) { + uint16_t count; + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = + const_CAST_bitset(ra->containers[i]); + memcpy(bitset_zone, bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + if (bc->cardinality != BITSET_UNKNOWN_CARDINALITY) { + count = (uint16_t)(bc->cardinality - 1); + } else { + count = + (uint16_t)(bitset_container_compute_cardinality(bc) - + 1); + } + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + size_t num_bytes = rc->n_runs * sizeof(rle16_t); + memcpy(run_zone, rc->runs, num_bytes); + run_zone += rc->n_runs; + count = (uint16_t)rc->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + size_t num_bytes = ac->cardinality * sizeof(uint16_t); + memcpy(array_zone, ac->array, num_bytes); + array_zone += ac->cardinality; + count = (uint16_t)(ac->cardinality - 1); + break; + } + default: + roaring_unreachable; } + memcpy(&count_zone[i], &count, 2); } + memcpy(key_zone, ra->keys, ra->size * sizeof(uint16_t)); + memcpy(typecode_zone, ra->typecodes, ra->size * sizeof(uint8_t)); + uint32_t header = ((uint32_t)ra->size << 15) | FROZEN_COOKIE; + memcpy(header_zone, &header, 4); } -static void offset_append_with_merge(roaring_array_t *ra, int k, container_t *c, uint8_t t) { - int size = ra_get_size(ra); - if (size == 0 || ra_get_key_at_index(ra, size-1) != k) { - // No merge. - ra_append(ra, k, c, t); - return; +const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, + size_t length) { + if ((uintptr_t)buf % 32 != 0) { + return NULL; } - uint8_t last_t, new_t; - container_t *last_c, *new_c; - - // NOTE: we don't need to unwrap here, since we added last_c ourselves - // we have the certainty it's not a shared container. - // The same applies to c, as it's the result of calling container_offset. - last_c = ra_get_container_at_index(ra, size-1, &last_t); - new_c = container_ior(last_c, last_t, c, t, &new_t); + // cookie and num_containers + if (length < 4) { + return NULL; + } + uint32_t header; + memcpy(&header, buf + length - 4, 4); // header may be misaligned + if ((header & 0x7FFF) != FROZEN_COOKIE) { + return NULL; + } + int32_t num_containers = (header >> 15); - ra_set_container_at_index(ra, size-1, new_c, new_t); + // typecodes, counts and keys + if (length < 4 + (size_t)num_containers * (1 + 2 + 2)) { + return NULL; + } + uint16_t *keys = (uint16_t *)(buf + length - 4 - num_containers * 5); + uint16_t *counts = (uint16_t *)(buf + length - 4 - num_containers * 3); + uint8_t *typecodes = (uint8_t *)(buf + length - 4 - num_containers * 1); - // Comparison of pointers of different origin is UB (or so claim some compiler - // makers), so we compare their bit representation only. - if ((uintptr_t)last_c != (uintptr_t)new_c) { - container_free(last_c, last_t); + // {bitset,array,run}_zone + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: + num_bitset_containers++; + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + case RUN_CONTAINER_TYPE: + num_run_containers++; + run_zone_size += counts[i] * sizeof(rle16_t); + break; + case ARRAY_CONTAINER_TYPE: + num_array_containers++; + array_zone_size += (counts[i] + UINT32_C(1)) * sizeof(uint16_t); + break; + default: + return NULL; + } } - container_free(c, t); -} + if (length != bitset_zone_size + run_zone_size + array_zone_size + + 5 * num_containers + 4) { + return NULL; + } + uint64_t *bitset_zone = (uint64_t *)(buf); + rle16_t *run_zone = (rle16_t *)(buf + bitset_zone_size); + uint16_t *array_zone = (uint16_t *)(buf + bitset_zone_size + run_zone_size); -// roaring_bitmap_add_offset adds the value 'offset' to each and every value in -// a bitmap, generating a new bitmap in the process. If offset + element is -// outside of the range [0,2^32), that the element will be dropped. -// We need "offset" to be 64 bits because we want to support values -// between -0xFFFFFFFF up to +0xFFFFFFFF. -roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, - int64_t offset) { - roaring_bitmap_t *answer; - roaring_array_t *ans_ra; - int64_t container_offset; - uint16_t in_offset; + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); - const roaring_array_t *bm_ra = &bm->high_low_container; - int length = bm_ra->size; + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; + } - if (offset == 0) { - return roaring_bitmap_copy(bm); + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.keys = (uint16_t *)keys; + rb->high_low_container.typecodes = (uint8_t *)typecodes; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); + // Ensure offset of high_low_container.containers is known distance used in + // C++ wrapper. sizeof(roaring_bitmap_t) is used as it is the size of the + // only allocation that precedes high_low_container.containers. If this is + // changed (new allocation or changed order), this offset will also need to + // be changed in the C++ wrapper. + assert(rb == + (roaring_bitmap_t *)((char *)rb->high_low_container.containers - + sizeof(roaring_bitmap_t))); + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + bitset->words = bitset_zone; + bitset->cardinality = counts[i] + UINT32_C(1); + rb->high_low_container.containers[i] = bitset; + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + break; + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = (run_container_t *)arena_alloc( + &arena, sizeof(run_container_t)); + run->capacity = counts[i]; + run->n_runs = counts[i]; + run->runs = run_zone; + rb->high_low_container.containers[i] = run; + run_zone += run->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + array->capacity = counts[i] + UINT32_C(1); + array->cardinality = counts[i] + UINT32_C(1); + array->array = array_zone; + rb->high_low_container.containers[i] = array; + array_zone += counts[i] + UINT32_C(1); + break; + } + default: + roaring_free(arena); + return NULL; + } } - container_offset = offset >> 16; - in_offset = (uint16_t)(offset - container_offset * (1 << 16)); - - answer = roaring_bitmap_create(); - roaring_bitmap_set_copy_on_write(answer, is_cow(bm)); + return rb; +} - ans_ra = &answer->high_low_container; +ALLOW_UNALIGNED +roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { + char *start_of_buf = (char *)buf; + uint32_t cookie; + int32_t num_containers; + uint16_t *descriptive_headers; + uint32_t *offset_headers = NULL; + const char *run_flag_bitset = NULL; + bool hasrun = false; - if (in_offset == 0) { - ans_ra = &answer->high_low_container; + // deserialize cookie + memcpy(&cookie, buf, sizeof(uint32_t)); + buf += sizeof(uint32_t); + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + memcpy(&num_containers, buf, sizeof(int32_t)); + buf += sizeof(int32_t); + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + run_flag_bitset = buf; + buf += run_flag_bitset_size; + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + if (num_containers >= NO_OFFSET_THRESHOLD) { + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); + } + } else { + return NULL; + } - for (int i = 0, j = 0; i < length; ++i) { - int64_t key = ra_get_key_at_index(bm_ra, i); - key += container_offset; + // calculate total size for allocation + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; - if (key < 0 || key >= (1 << 16)) { - continue; + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + uint32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; } + } - ra_append_copy(ans_ra, bm_ra, i, false); - ans_ra->keys[j++] = key; + if (isbitmap) { + num_bitset_containers++; + } else if (isrun) { + num_run_containers++; + } else { + num_array_containers++; } + } - return answer; + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); + alloc_size += num_containers * sizeof(uint16_t); // keys + alloc_size += num_containers * sizeof(uint8_t); // typecodes + + // allocate bitmap and construct containers + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; } - uint8_t t; - const container_t *c; - container_t *lo, *hi, **lo_ptr, **hi_ptr; - int64_t k; + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); - for (int i = 0; i < length; ++i) { - lo = hi = NULL; - lo_ptr = hi_ptr = NULL; + uint16_t *keys = + (uint16_t *)arena_alloc(&arena, num_containers * sizeof(uint16_t)); + uint8_t *typecodes = + (uint8_t *)arena_alloc(&arena, num_containers * sizeof(uint8_t)); - k = ra_get_key_at_index(bm_ra, i)+container_offset; - if (k >= 0 && k < (1 << 16)) { - lo_ptr = &lo; - } - if (k+1 >= 0 && k+1 < (1 << 16)) { - hi_ptr = &hi; - } - if (lo_ptr == NULL && hi_ptr == NULL) { - continue; + rb->high_low_container.keys = keys; + rb->high_low_container.typecodes = typecodes; + + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + int32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } } - c = ra_get_container_at_index(bm_ra, i, &t); - c = container_unwrap_shared(c, &t); + keys[i] = descriptive_headers[2 * i]; - container_add_offset(c, t, lo_ptr, hi_ptr, in_offset); - if (lo != NULL) { - offset_append_with_merge(ans_ra, k, lo, t); - } - if (hi != NULL) { - ra_append(ans_ra, k+1, hi, t); + if (isbitmap) { + typecodes[i] = BITSET_CONTAINER_TYPE; + bitset_container_t *c = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + c->cardinality = cardinality; + if (offset_headers != NULL) { + c->words = (uint64_t *)(start_of_buf + offset_headers[i]); + } else { + c->words = (uint64_t *)buf; + buf += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } + rb->high_low_container.containers[i] = c; + } else if (isrun) { + typecodes[i] = RUN_CONTAINER_TYPE; + run_container_t *c = + (run_container_t *)arena_alloc(&arena, sizeof(run_container_t)); + c->capacity = cardinality; + uint16_t n_runs; + if (offset_headers != NULL) { + memcpy(&n_runs, start_of_buf + offset_headers[i], + sizeof(uint16_t)); + c->n_runs = n_runs; + c->runs = (rle16_t *)(start_of_buf + offset_headers[i] + + sizeof(uint16_t)); + } else { + memcpy(&n_runs, buf, sizeof(uint16_t)); + c->n_runs = n_runs; + buf += sizeof(uint16_t); + c->runs = (rle16_t *)buf; + buf += c->n_runs * sizeof(rle16_t); + } + rb->high_low_container.containers[i] = c; + } else { + typecodes[i] = ARRAY_CONTAINER_TYPE; + array_container_t *c = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + c->cardinality = cardinality; + c->capacity = cardinality; + if (offset_headers != NULL) { + c->array = (uint16_t *)(start_of_buf + offset_headers[i]); + } else { + c->array = (uint16_t *)buf; + buf += cardinality * sizeof(uint16_t); + } + rb->high_low_container.containers[i] = c; } } - return answer; + return rb; } -roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2, - const bool bitsetconversion) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); - } - if (0 == length2) { - return roaring_bitmap_copy(x1); +bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { + uint32_t max_value = roaring_bitmap_maximum(r); + size_t new_array_size = (size_t)(((uint64_t)max_value + 63) / 64); + bool resize_ok = bitset_resize(bitset, new_array_size, true); + if (!resize_ok) { + return false; } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c; - if (bitsetconversion && - (get_container_type(c1, type1) != BITSET_CONTAINER_TYPE) && - (get_container_type(c2, type2) != BITSET_CONTAINER_TYPE) - ){ - container_t *newc1 = - container_mutable_unwrap_shared(c1, &type1); - newc1 = container_to_bitset(newc1, type1); - type1 = BITSET_CONTAINER_TYPE; - c = container_lazy_ior(newc1, type1, c2, type2, - &result_type); - if (c != newc1) { // should not happen - container_free(newc1, type1); + const roaring_array_t *ra = &r->high_low_container; + for (int i = 0; i < ra->size; ++i) { + uint64_t *words = bitset->array + (ra->keys[i] << 10); + uint8_t type = ra->typecodes[i]; + const container_t *c = ra->containers[i]; + if (type == SHARED_CONTAINER_TYPE) { + c = container_unwrap_shared(c, &type); + } + switch (type) { + case BITSET_CONTAINER_TYPE: { + size_t max_word_index = new_array_size - (ra->keys[i] << 10); + if (max_word_index > 1024) { + max_word_index = 1024; } - } else { - c = container_lazy_or(c1, type1, c2, type2, &result_type); - } - // since we assume that the initial containers are non-empty, - // the - // result here - // can only be non-empty - ra_append(&answer->high_low_container, s1, c, result_type); - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + const bitset_container_t *src = const_CAST_bitset(c); + memcpy(words, src->words, max_word_index * sizeof(uint64_t)); + } break; + case ARRAY_CONTAINER_TYPE: { + const array_container_t *src = const_CAST_array(c); + bitset_set_list(words, src->array, src->cardinality); + } break; + case RUN_CONTAINER_TYPE: { + const run_container_t *src = const_CAST_run(c); + for (int32_t rlepos = 0; rlepos < src->n_runs; ++rlepos) { + rle16_t rle = src->runs[rlepos]; + bitset_set_lenrange(words, rle.value, rle.length); + } + } break; + default: + roaring_unreachable; } } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; + return true; } -void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2, - const bool bitsetconversion) { - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { +#endif +/* end file src/roaring.c */ +/* begin file src/roaring64.c */ +#include +#include +#include +#include - if (0 == length2) return; - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; - } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - if (!container_is_full(c1, type1)) { - if ((bitsetconversion == false) || - (get_container_type(c1, type1) == BITSET_CONTAINER_TYPE) - ){ - c1 = get_writable_copy_if_shared(c1, &type1); - } else { - // convert to bitset - container_t *old_c1 = c1; - uint8_t old_type1 = type1; - c1 = container_mutable_unwrap_shared(c1, &type1); - c1 = container_to_bitset(c1, type1); - container_free(old_c1, old_type1); - type1 = BITSET_CONTAINER_TYPE; - } +// For serialization / deserialization +// containers.h last to avoid conflict with ROARING_CONTAINER_T. - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_lazy_ior(c1, type1, c2, type2, - &result_type); +#ifdef __cplusplus +using namespace ::roaring::internal; - if (c != c1) { // in this instance a new container was created, - // and we need to free the old one - container_free(c1, type1); - } +extern "C" { +namespace roaring { +namespace api { +#endif - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +// TODO: Copy on write. +// TODO: Error on failed allocation. - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); +typedef struct roaring64_bitmap_s { + art_t art; + uint8_t flags; +} roaring64_bitmap_t; - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - // container_t *c2_clone = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +// Leaf type of the ART used to keep the high 48 bits of each entry. +typedef struct roaring64_leaf_s { + art_val_t _pad; + uint8_t typecode; + container_t *container; +} roaring64_leaf_t; + +// Alias to make it easier to work with, since it's an internal-only type +// anyway. +typedef struct roaring64_leaf_s leaf_t; + +// Iterator struct to hold iteration state. +typedef struct roaring64_iterator_s { + const roaring64_bitmap_t *parent; + art_iterator_t art_it; + roaring_container_iterator_t container_it; + uint64_t high48; // Key that art_it points to. + + uint64_t value; + bool has_value; + + // If has_value is false, then the iterator is saturated. This field + // indicates the direction of saturation. If true, there are no more values + // in the forward direction. If false, there are no more values in the + // backward direction. + bool saturated_forward; +} roaring64_iterator_t; + +// Splits the given uint64 key into high 48 bit and low 16 bit components. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { + uint64_t tmp = croaring_htobe64(key); + memcpy(high48_out, (uint8_t *)(&tmp), ART_KEY_BYTES); + return (uint16_t)key; +} + +// Recombines the high 48 bit and low 16 bit components into a uint64 key. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint64_t combine_key(const uint8_t high48[], uint16_t low16) { + uint64_t result = 0; + memcpy((uint8_t *)(&result), high48, ART_KEY_BYTES); + return croaring_be64toh(result) | low16; +} + +static inline uint64_t minimum(uint64_t a, uint64_t b) { + return (a < b) ? a : b; +} + +static inline leaf_t *create_leaf(container_t *container, uint8_t typecode) { + leaf_t *leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf->container = container; + leaf->typecode = typecode; + return leaf; +} + +static inline leaf_t *copy_leaf_container(const leaf_t *leaf) { + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->typecode = leaf->typecode; + // get_copy_of_container modifies the typecode passed in. + result_leaf->container = get_copy_of_container( + leaf->container, &result_leaf->typecode, /*copy_on_write=*/false); + return result_leaf; +} + +static inline void free_leaf(leaf_t *leaf) { roaring_free(leaf); } + +static inline int compare_high48(art_key_chunk_t key1[], + art_key_chunk_t key2[]) { + return art_compare_keys(key1, key2); +} + +static inline bool roaring64_iterator_init_at_leaf_first( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = 0; + it->container_it = + container_init_iterator(leaf->container, leaf->typecode, &low16); + it->value = it->high48 | low16; + return (it->has_value = true); +} + +static inline bool roaring64_iterator_init_at_leaf_last( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = 0; + it->container_it = + container_init_iterator_last(leaf->container, leaf->typecode, &low16); + it->value = it->high48 | low16; + return (it->has_value = true); +} + +static inline roaring64_iterator_t *roaring64_iterator_init_at( + const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { + it->parent = r; + it->art_it = art_init_iterator(&r->art, first); + it->has_value = it->art_it.value != NULL; + if (it->has_value) { + if (first) { + roaring64_iterator_init_at_leaf_first(it); + } else { + roaring64_iterator_init_at_leaf_last(it); } + } else { + it->saturated_forward = first; } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); - } + return it; } -roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); +roaring64_bitmap_t *roaring64_bitmap_create(void) { + roaring64_bitmap_t *r = + (roaring64_bitmap_t *)roaring_malloc(sizeof(roaring64_bitmap_t)); + r->art.root = NULL; + r->flags = 0; + return r; +} + +void roaring64_bitmap_free(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + art_iterator_next(&it); } - if (0 == length2) { - return roaring_bitmap_copy(x1); + art_free(&r->art); + roaring_free(r); +} + +roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint8_t result_typecode = leaf->typecode; + container_t *result_container = get_copy_of_container( + leaf->container, &result_typecode, /*copy_on_write=*/false); + leaf_t *result_leaf = create_leaf(result_container, result_typecode); + art_insert(&result->art, it.key, (art_val_t *)result_leaf); + art_iterator_next(&it); } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - container_t *c = container_lazy_xor( - c1, type1, c2, type2, &result_type); + return result; +} - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); +roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, + uint64_t step) { + if (step == 0 || max <= min) { + return NULL; + } + roaring64_bitmap_t *r = roaring64_bitmap_create(); + if (step >= (1 << 16)) { + // Only one value per container. + for (uint64_t value = min; value < max; value += step) { + roaring64_bitmap_add(r, value); + if (value > UINT64_MAX - step) { + break; } + } + return r; + } + do { + uint64_t high_bits = min & 0xFFFFFFFFFFFF0000; + uint16_t container_min = min & 0xFFFF; + uint32_t container_max = (uint32_t)minimum(max - high_bits, 1 << 16); - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint8_t typecode; + container_t *container = container_from_range( + &typecode, container_min, container_max, (uint16_t)step); - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); + uint8_t high48[ART_KEY_BYTES]; + split_key(min, high48); + leaf_t *leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint64_t gap = container_max - container_min + step - 1; + uint64_t increment = gap - (gap % step); + if (min > UINT64_MAX - increment) { + break; } + min += increment; + } while (min < max); + return r; +} + +roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, + const uint64_t *vals) { + roaring64_bitmap_t *r = roaring64_bitmap_create(); + roaring64_bitmap_add_many(r, n_args, vals); + return r; +} + +roaring64_bitmap_t *roaring64_bitmap_of(size_t n_args, ...) { + roaring64_bitmap_t *r = roaring64_bitmap_create(); + roaring64_bulk_context_t context = {0}; + va_list ap; + va_start(ap, n_args); + for (size_t i = 0; i < n_args; i++) { + uint64_t val = va_arg(ap, uint64_t); + roaring64_bitmap_add_bulk(r, &context, val); } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); + va_end(ap); + return r; +} + +static inline leaf_t *containerptr_roaring64_bitmap_add(roaring64_bitmap_t *r, + uint8_t *high48, + uint16_t low16, + leaf_t *leaf) { + if (leaf != NULL) { + uint8_t typecode2; + container_t *container2 = + container_add(leaf->container, low16, leaf->typecode, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + leaf->container = container2; + leaf->typecode = typecode2; + } + return leaf; + } else { + array_container_t *ac = array_container_create(); + uint8_t typecode; + container_t *container = + container_add(ac, low16, ARRAY_CONTAINER_TYPE, &typecode); + assert(ac == container); + leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); + return leaf; + } +} + +void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); +} + +bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + + int old_cardinality = 0; + if (leaf != NULL) { + old_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + } + leaf = containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + int new_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + return old_cardinality != new_cardinality; +} + +void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + if (context->leaf != NULL && + compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode2; + container_t *container2 = + container_add(context->leaf->container, low16, + context->leaf->typecode, &typecode2); + if (container2 != context->leaf->container) { + container_free(context->leaf->container, context->leaf->typecode); + context->leaf->container = container2; + context->leaf->typecode = typecode2; + } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + context->leaf = + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + memcpy(context->high_bytes, high48, ART_KEY_BYTES); } - return answer; } -void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = {0}; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_add_bulk(r, &context, *current_val); + } +} - if (0 == length2) return; +static inline void add_range_closed_at(art_t *art, uint8_t *high48, + uint16_t min, uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf != NULL) { + uint8_t typecode2; + container_t *container2 = container_add_range( + leaf->container, leaf->typecode, min, max, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + leaf->container = container2; + leaf->typecode = typecode2; + } + return; + } + uint8_t typecode; + // container_add_range is inclusive, but `container_range_of_ones` is + // exclusive. + container_t *container = container_range_of_ones(min, max + 1, &typecode); + leaf = create_leaf(container, typecode); + art_insert(art, high48, (art_val_t *)leaf); +} - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); +void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { return; } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); + roaring64_bitmap_add_range_closed(r, min, max - 1); +} - // We do the computation "in place" only when c1 is not a shared container. - // Rationale: using a shared container safely with in place computation would - // require making a copy and then doing the computation in place which is likely - // less efficient than avoiding in place entirely and always generating a new - // container. +void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_lazy_xor(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // release - } - else { - c = container_lazy_ixor(c1, type1, c2, type2, &result_type); - } + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only populate range within one container. + add_range_closed_at(art, min_high48, min_low16, max_low16); + return; + } - if (container_nonzero_cardinality(c, result_type)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - ++pos1; - } else { - container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; - } - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + // Populate a range across containers. Fill intermediate containers + // entirely. + add_range_closed_at(art, min_high48, min_low16, 0xffff); + uint64_t min_high_bits = min >> 16; + uint64_t max_high_bits = max >> 16; + for (uint64_t current = min_high_bits + 1; current < max_high_bits; + ++current) { + uint8_t current_high48[ART_KEY_BYTES]; + split_key(current << 16, current_high48); + add_range_closed_at(art, current_high48, 0, 0xffff); + } + add_range_closed_at(art, max_high48, 0, max_low16); +} - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); +bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf != NULL) { + return container_contains(leaf->container, low16, leaf->typecode); + } + return false; +} - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - // container_t *c2_clone = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); - } +bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return true; } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); + + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + uint64_t max_high48_bits = (max - 1) & 0xFFFFFFFFFFFF0000; // Inclusive + + art_iterator_t it = art_lower_bound(&r->art, min_high48); + if (it.value == NULL || combine_key(it.key, 0) > min) { + return false; } -} + uint64_t prev_high48_bits = min & 0xFFFFFFFFFFFF0000; + while (it.value != NULL) { + uint64_t current_high48_bits = combine_key(it.key, 0); + if (current_high48_bits > max_high48_bits) { + // We've passed the end of the range with all containers containing + // the range. + return true; + } + if (current_high48_bits - prev_high48_bits > 0x10000) { + // There is a gap in the iterator that falls in the range. + return false; + } -void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r) { - roaring_array_t *ra = &r->high_low_container; + leaf_t *leaf = (leaf_t *)it.value; + uint32_t container_min = 0; + if (compare_high48(it.key, min_high48) == 0) { + container_min = min_low16; + } + uint32_t container_max = 0xFFFF + 1; // Exclusive + if (compare_high48(it.key, max_high48) == 0) { + container_max = max_low16; + } - for (int i = 0; i < ra->size; ++i) { - const uint8_t old_type = ra->typecodes[i]; - container_t *old_c = ra->containers[i]; - uint8_t new_type = old_type; - container_t *new_c = container_repair_after_lazy(old_c, &new_type); - ra->containers[i] = new_c; - ra->typecodes[i] = new_type; + // For the first and last containers we use container_contains_range, + // for the intermediate containers we can use container_is_full. + if (container_min == 0 && container_max == 0xFFFF + 1) { + if (!container_is_full(leaf->container, leaf->typecode)) { + return false; + } + } else if (!container_contains_range(leaf->container, container_min, + container_max, leaf->typecode)) { + return false; + } + prev_high48_bits = current_high48_bits; + art_iterator_next(&it); } + return prev_high48_bits == max_high48_bits; } +bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - -/** -* roaring_bitmap_rank returns the number of integers that are smaller or equal -* to x. -*/ -uint64_t roaring_bitmap_rank(const roaring_bitmap_t *bm, uint32_t x) { - uint64_t size = 0; - uint32_t xhigh = x >> 16; - for (int i = 0; i < bm->high_low_container.size; i++) { - uint32_t key = bm->high_low_container.keys[i]; - if (xhigh > key) { - size += - container_get_cardinality(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i]); - } else if (xhigh == key) { - return size + container_rank(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i], - x & 0xFFFF); + if (context->leaf == NULL || context->high_bytes != high48) { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf == NULL) { + return false; + } + context->leaf = leaf; + memcpy(context->high_bytes, high48, ART_KEY_BYTES); + } + return container_contains(context->leaf->container, low16, + context->leaf->typecode); +} + +bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, + uint64_t *element) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t start_rank = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint64_t cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + if (start_rank + cardinality > rank) { + uint32_t uint32_start = 0; + uint32_t uint32_rank = rank - start_rank; + uint32_t uint32_element = 0; + if (container_select(leaf->container, leaf->typecode, &uint32_start, + uint32_rank, &uint32_element)) { + *element = combine_key(it.key, (uint16_t)uint32_element); + return true; + } + return false; + } + start_rank += cardinality; + art_iterator_next(&it); + } + return false; +} + +uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t rank = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + rank += container_get_cardinality(leaf->container, leaf->typecode); + } else if (compare_result == 0) { + return rank + + container_rank(leaf->container, leaf->typecode, low16); } else { - return size; + return rank; } + art_iterator_next(&it); } - return size; + return rank; } -/** - * roaring_bitmap_get_index returns the index of x, if not exsist return -1. - */ -int64_t roaring_bitmap_get_index(const roaring_bitmap_t *bm, uint32_t x) { - int64_t index = 0; - const uint16_t xhigh = x >> 16; - int32_t high_idx = ra_get_index(&bm->high_low_container, xhigh); - if (high_idx < 0) return -1; +bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, + uint64_t *out_index) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - for (int i = 0; i < bm->high_low_container.size; i++) { - uint32_t key = bm->high_low_container.keys[i]; - if (xhigh > key) { - index += - container_get_cardinality(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i]); - } else if (xhigh == key) { - int32_t low_idx = container_get_index( - bm->high_low_container.containers[high_idx], - bm->high_low_container.typecodes[high_idx], x & 0xFFFF); - if (low_idx < 0) return -1; - return index + low_idx; + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t index = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + index += container_get_cardinality(leaf->container, leaf->typecode); + } else if (compare_result == 0) { + int index16 = + container_get_index(leaf->container, leaf->typecode, low16); + if (index16 < 0) { + return false; + } + *out_index = index + index16; + return true; } else { - return -1; + return false; } + art_iterator_next(&it); } - return index; + return false; } -/** -* roaring_bitmap_smallest returns the smallest value in the set. -* Returns UINT32_MAX if the set is empty. -*/ -uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *bm) { - if (bm->high_low_container.size > 0) { - container_t *c = bm->high_low_container.containers[0]; - uint8_t type = bm->high_low_container.typecodes[0]; - uint32_t key = bm->high_low_container.keys[0]; - uint32_t lowvalue = container_minimum(c, type); - return lowvalue | (key << 16); +static inline leaf_t *containerptr_roaring64_bitmap_remove( + roaring64_bitmap_t *r, uint8_t *high48, uint16_t low16, leaf_t *leaf) { + if (leaf == NULL) { + return NULL; } - return UINT32_MAX; -} -/** -* roaring_bitmap_smallest returns the greatest value in the set. -* Returns 0 if the set is empty. -*/ -uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *bm) { - if (bm->high_low_container.size > 0) { - container_t *container = - bm->high_low_container.containers[bm->high_low_container.size - 1]; - uint8_t typecode = - bm->high_low_container.typecodes[bm->high_low_container.size - 1]; - uint32_t key = - bm->high_low_container.keys[bm->high_low_container.size - 1]; - uint32_t lowvalue = container_maximum(container, typecode); - return lowvalue | (key << 16); + container_t *container = leaf->container; + uint8_t typecode = leaf->typecode; + uint8_t typecode2; + container_t *container2 = + container_remove(container, low16, typecode, &typecode2); + if (container2 != container) { + container_free(container, typecode); + leaf->container = container2; + leaf->typecode = typecode2; } - return 0; + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + leaf = (leaf_t *)art_erase(&r->art, high48); + if (leaf != NULL) { + free_leaf(leaf); + } + return NULL; + } + return leaf; } -bool roaring_bitmap_select(const roaring_bitmap_t *bm, uint32_t rank, - uint32_t *element) { - container_t *container; - uint8_t typecode; - uint16_t key; - uint32_t start_rank = 0; - int i = 0; - bool valid = false; - while (!valid && i < bm->high_low_container.size) { - container = bm->high_low_container.containers[i]; - typecode = bm->high_low_container.typecodes[i]; - valid = - container_select(container, typecode, &start_rank, rank, element); - i++; - } +void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - if (valid) { - key = bm->high_low_container.keys[i - 1]; - *element |= (((uint32_t)key) << 16); // w/o cast, key promotes signed - return true; - } else - return false; + leaf_t *leaf = (leaf_t *)art_find(art, high48); + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); } -bool roaring_bitmap_intersect(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - uint64_t answer = 0; - int pos1 = 0, pos2 = 0; +bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(art, high48); - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(& x1->high_low_container, pos1); - const uint16_t s2 = ra_get_key_at_index(& x2->high_low_container, pos2); + if (leaf == NULL) { + return false; + } + int old_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + leaf = containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); + if (leaf == NULL) { + return true; + } + int new_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + return new_cardinality != old_cardinality; +} + +void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + if (context->leaf != NULL && + compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode2; + container_t *container2 = + container_remove(context->leaf->container, low16, + context->leaf->typecode, &typecode2); + if (container2 != context->leaf->container) { + container_free(context->leaf->container, context->leaf->typecode); + context->leaf->container = container2; + context->leaf->typecode = typecode2; + } + if (!container_nonzero_cardinality(container2, typecode2)) { + leaf_t *leaf = (leaf_t *)art_erase(art, high48); + container_free(container2, typecode2); + free_leaf(leaf); + } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(art, high48); + context->leaf = + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); + memcpy(context->high_bytes, high48, ART_KEY_BYTES); + } +} - if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - if (container_intersect(c1, type1, c2, type2)) - return true; - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(& x1->high_low_container, s2, pos1); - } else { // s1 > s2 - pos2 = ra_advance_until(& x2->high_low_container, s1, pos2); +void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = {0}; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_remove_bulk(r, &context, *current_val); + } +} + +static inline void remove_range_closed_at(art_t *art, uint8_t *high48, + uint16_t min, uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf == NULL) { + return; + } + uint8_t typecode2; + container_t *container2 = container_remove_range( + leaf->container, leaf->typecode, min, max, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + if (container2 != NULL) { + leaf->container = container2; + leaf->typecode = typecode2; + } else { + art_erase(art, high48); + free_leaf(leaf); } } - return answer != 0; } -bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, - uint64_t x, uint64_t y) { - if (x >= y) { - // Empty range. - return false; +void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; } - roaring_uint32_iterator_t it; - roaring_init_iterator(bm, &it); - if (!roaring_move_uint32_iterator_equalorlarger(&it, x)) { - // No values above x. - return false; + roaring64_bitmap_remove_range_closed(r, min, max - 1); +} + +void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; } - if (it.current_value >= y) { - // No values below y. - return false; + + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only remove a range within one container. + remove_range_closed_at(art, min_high48, min_low16, max_low16); + return; } - return true; + + // Remove a range across containers. Remove intermediate containers + // entirely. + remove_range_closed_at(art, min_high48, min_low16, 0xffff); + + art_iterator_t it = art_upper_bound(art, min_high48); + while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { + leaf_t *leaf = (leaf_t *)art_iterator_erase(art, &it); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + } + remove_range_closed_at(art, max_high48, 0, max_low16); } +uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t cardinality = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + cardinality += + container_get_cardinality(leaf->container, leaf->typecode); + art_iterator_next(&it); + } + return cardinality; +} -uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - uint64_t answer = 0; - int pos1 = 0, pos2 = 0; - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - const uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); +uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return 0; + } + max--; // A closed range is easier to work with. - if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); - answer += container_and_cardinality(c1, type1, c2, type2); - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + uint64_t cardinality = 0; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + + art_iterator_t it = art_lower_bound(&r->art, min_high48); + while (it.value != NULL) { + int max_compare_result = compare_high48(it.key, max_high48); + if (max_compare_result > 0) { + // We're outside the range. + break; + } + + leaf_t *leaf = (leaf_t *)it.value; + if (max_compare_result == 0) { + // We're at the max high key, add only the range up to the low + // 16 bits of max. + cardinality += + container_rank(leaf->container, leaf->typecode, max_low16); + } else { + // We're not yet at the max high key, add the full container + // range. + cardinality += + container_get_cardinality(leaf->container, leaf->typecode); } + if (compare_high48(it.key, min_high48) == 0 && min_low16 > 0) { + // We're at the min high key, remove the range up to the low 16 + // bits of min. + cardinality -= + container_rank(leaf->container, leaf->typecode, min_low16 - 1); + } + art_iterator_next(&it); } - return answer; + return cardinality; } -double roaring_bitmap_jaccard_index(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return (double)inter / (double)(c1 + c2 - inter); +bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r) { + return art_is_empty(&r->art); } -uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 + c2 - inter; +uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + if (it.value == NULL) { + return UINT64_MAX; + } + leaf_t *leaf = (leaf_t *)it.value; + return combine_key(it.key, + container_minimum(leaf->container, leaf->typecode)); } -uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 - inter; +uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/false); + if (it.value == NULL) { + return 0; + } + leaf_t *leaf = (leaf_t *)it.value; + return combine_key(it.key, + container_maximum(leaf->container, leaf->typecode)); } -uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 + c2 - 2 * inter; +bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool has_run_container = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint8_t new_typecode; + // We don't need to free the existing container if a new one was + // created, convert_run_optimize does that internally. + leaf->container = convert_run_optimize(leaf->container, leaf->typecode, + &new_typecode); + leaf->typecode = new_typecode; + has_run_container |= new_typecode == RUN_CONTAINER_TYPE; + art_iterator_next(&it); + } + return has_run_container; } +static bool roaring64_leaf_internal_validate(const art_val_t *val, + const char **reason) { + leaf_t *leaf = (leaf_t *)val; + return container_internal_validate(leaf->container, leaf->typecode, reason); +} -bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - /* - * the next function call involves a binary search and lots of branching. - */ - int32_t i = ra_get_index(&r->high_low_container, hb); - if (i < 0) return false; +bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, + const char **reason) { + return art_internal_validate(&r->art, reason, + roaring64_leaf_internal_validate); +} - uint8_t typecode; - // next call ought to be cheap - container_t *container = - ra_get_container_at_index(&r->high_low_container, i, &typecode); - // rest might be a tad expensive, possibly involving another round of binary search - return container_contains(container, val & 0xFFFF, typecode); +bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + if (compare_high48(it1.key, it2.key) != 0) { + return false; + } + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + if (!container_equals(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode)) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + return it1.value == NULL && it2.value == NULL; } +bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); -/** - * Check whether a range of values from range_start (included) to range_end (excluded) is present - */ -bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, uint64_t range_start, uint64_t range_end) { - if(range_end >= UINT64_C(0x100000000)) { - range_end = UINT64_C(0x100000000); + while (it1.value != NULL) { + bool it2_present = it2.value != NULL; + + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + if (!container_is_subset(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode)) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if (!it2_present || compare_result < 0) { + return false; + } else if (compare_result > 0) { + art_iterator_lower_bound(&it2, it1.key); + } } - if (range_start >= range_end) return true; // empty range are always contained! - if (range_end - range_start == 1) return roaring_bitmap_contains(r, (uint32_t)range_start); - uint16_t hb_rs = (uint16_t)(range_start >> 16); - uint16_t hb_re = (uint16_t)((range_end - 1) >> 16); - const int32_t span = hb_re - hb_rs; - const int32_t hlc_sz = ra_get_size(&r->high_low_container); - if (hlc_sz < span + 1) { - return false; + return true; +} + +bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + return roaring64_bitmap_get_cardinality(r1) < + roaring64_bitmap_get_cardinality(r2) && + roaring64_bitmap_is_subset(r1, r2); +} + +roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result_leaf->container = container_and( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } } - int32_t is = ra_get_index(&r->high_low_container, hb_rs); - int32_t ie = ra_get_index(&r->high_low_container, hb_re); - if ((ie < 0) || (is < 0) || ((ie - is) != span) || ie >= hlc_sz) { - return false; + return result; +} + +uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t result = 0; + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output cardinaltiy it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result += + container_and_cardinality(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } } - const uint32_t lb_rs = range_start & 0xFFFF; - const uint32_t lb_re = ((range_end - 1) & 0xFFFF) + 1; - uint8_t type; - container_t *c = ra_get_container_at_index(&r->high_low_container, is, - &type); - if (hb_rs == hb_re) { - return container_contains_range(c, lb_rs, lb_re, type); + return result; +} + +// Inplace and (modifies its first argument). +void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { + return; } - if (!container_contains_range(c, lb_rs, 1 << 16, type)) { - return false; + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. !it2_present -> erase it1 + // 2. it2_present + // a. it1 < it2 -> erase it1 + // b. it1 == it2 -> output it1 & it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2a: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + + // We do the computation "in place" only when c1 is not a + // shared container. Rationale: using a shared container + // safely with in place computation would require making a + // copy and then doing the computation in place which is + // likely less efficient than avoiding in place entirely and + // always generating a new container. + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_and( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } else { + container2 = container_iand( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + + if (container2 != leaf1->container) { + container_free(leaf1->container, leaf1->typecode); + leaf1->container = container2; + leaf1->typecode = typecode2; + } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); + } + } + + if (!it2_present || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *leaf = (leaf_t *)art_iterator_erase(&r1->art, &it1); + assert(leaf != NULL); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_lower_bound(&it2, it1.key); + } + } +} + +bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + bool intersect = false; + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> intersect |= it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + intersect |= container_intersect(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } } - c = ra_get_container_at_index(&r->high_low_container, ie, &type); - if (!container_contains_range(c, 0, lb_re, type)) { + return intersect; +} + +bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { return false; } - for (int32_t i = is + 1; i < ie; ++i) { - c = ra_get_container_at_index(&r->high_low_container, i, &type); - if (!container_is_full(c, type) ) { - return false; - } + roaring64_iterator_t it; + roaring64_iterator_init_at(r, &it, /*first=*/true); + if (!roaring64_iterator_move_equalorlarger(&it, min)) { + return false; } - return true; + return roaring64_iterator_has_value(&it) && + roaring64_iterator_value(&it) < max; } - -bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - return (roaring_bitmap_get_cardinality(r2) > - roaring_bitmap_get_cardinality(r1) && - roaring_bitmap_is_subset(r1, r2)); +double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return (double)inter / (double)(c1 + c2 - inter); } +roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 | it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->container = container_or( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_insert(&result->art, it2.key, (art_val_t *)result_leaf); + art_iterator_next(&it2); + } + } + return result; +} -/* - * FROZEN SERIALIZATION FORMAT DESCRIPTION - * - * -- (beginning must be aligned by 32 bytes) -- - * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * num_bitset_containers] - * rle16_t[total number of rle elements in all run containers] - * uint16_t[total number of array elements in all array containers] - * uint16_t[num_containers] - * uint16_t[num_containers] - * uint8_t[num_containers] - *
uint32_t - * - *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) - * and the number of containers (17 bits). - * - * stores number of elements for every container. - * Its meaning depends on container type. - * For array and bitset containers, this value is the container cardinality minus one. - * For run container, it is the number of rle_t elements (n_runs). - * - * ,, are flat arrays of elements of - * all containers of respective type. - * - * <*_data> and are kept close together because they are not accessed - * during deserilization. This may reduce IO in case of large mmaped bitmaps. - * All members have their native alignments during deserilization except
, - * which is not guaranteed to be aligned by 4 bytes. - */ +uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - inter; +} -size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *rb) { - const roaring_array_t *ra = &rb->high_low_container; - size_t num_bytes = 0; - for (int32_t i = 0; i < ra->size; i++) { - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - num_bytes += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - num_bytes += rc->n_runs * sizeof(rle16_t); - break; +void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { + return; + } + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 | it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_or(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode, + &typecode2); + } else { + container2 = container_ior( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + if (container2 != leaf1->container) { + container_free(leaf1->container, leaf1->typecode); + leaf1->container = container2; + leaf1->typecode = typecode2; + } + art_iterator_next(&it1); + art_iterator_next(&it2); } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - num_bytes += ac->cardinality * sizeof(uint16_t); - break; + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_iterator_insert(&r1->art, &it1, it2.key, + (art_val_t *)result_leaf); + art_iterator_next(&it2); + } + } +} + +roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->container = container_xor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, + result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); } - default: - roaring_unreachable; + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_insert(&result->art, it2.key, (art_val_t *)result_leaf); + art_iterator_next(&it2); } } - num_bytes += (2 + 2 + 1) * ra->size; // keys, counts, typecodes - num_bytes += 4; // header - return num_bytes; + return result; } -inline static void *arena_alloc(char **arena, size_t num_bytes) { - char *res = *arena; - *arena += num_bytes; - return res; +uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - 2 * inter; } -void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *rb, char *buf) { - /* - * Note: we do not require user to supply a specifically aligned buffer. - * Thus we have to use memcpy() everywhere. - */ - - const roaring_array_t *ra = &rb->high_low_container; +void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + assert(r1 != r2); + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + container_t *container1 = leaf1->container; + uint8_t typecode1 = leaf1->typecode; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_xor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + if (container2 != container1) { + // We only free when doing container_xor, not + // container_ixor, as ixor frees the original + // internally. + container_free(container1, typecode1); + } + } else { + container2 = container_ixor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + leaf1->container = container2; + leaf1->typecode = typecode2; - size_t bitset_zone_size = 0; - size_t run_zone_size = 0; - size_t array_zone_size = 0; - for (int32_t i = 0; i < ra->size; i++) { - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - bitset_zone_size += - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - run_zone_size += rc->n_runs * sizeof(rle16_t); - break; + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + if (it1_present) { + art_iterator_insert(&r1->art, &it1, it2.key, + (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else { + art_insert(&r1->art, it2.key, (art_val_t *)result_leaf); } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - array_zone_size += ac->cardinality * sizeof(uint16_t); - break; + art_iterator_next(&it2); + } + } +} + +roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result_leaf->container = container_andnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, + result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); } - default: - roaring_unreachable; + } + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); } } + return result; +} - uint64_t *bitset_zone = (uint64_t *)arena_alloc(&buf, bitset_zone_size); - rle16_t *run_zone = (rle16_t *)arena_alloc(&buf, run_zone_size); - uint16_t *array_zone = (uint16_t *)arena_alloc(&buf, array_zone_size); - uint16_t *key_zone = (uint16_t *)arena_alloc(&buf, 2*ra->size); - uint16_t *count_zone = (uint16_t *)arena_alloc(&buf, 2*ra->size); - uint8_t *typecode_zone = (uint8_t *)arena_alloc(&buf, ra->size); - uint32_t *header_zone = (uint32_t *)arena_alloc(&buf, 4); +uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 - inter; +} - for (int32_t i = 0; i < ra->size; i++) { - uint16_t count; - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - const bitset_container_t *bc = - const_CAST_bitset(ra->containers[i]); - memcpy(bitset_zone, bc->words, - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); - bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; - if (bc->cardinality != BITSET_UNKNOWN_CARDINALITY) { - count = bc->cardinality - 1; +void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + container_t *container1 = leaf1->container; + uint8_t typecode1 = leaf1->typecode; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_andnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + if (container2 != container1) { + // We only free when doing container_andnot, not + // container_iandnot, as iandnot frees the original + // internally. + container_free(container1, typecode1); + } } else { - count = bitset_container_compute_cardinality(bc) - 1; + container2 = container_iandnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); } - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - size_t num_bytes = rc->n_runs * sizeof(rle16_t); - memcpy(run_zone, rc->runs, num_bytes); - run_zone += rc->n_runs; - count = rc->n_runs; - break; - } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - size_t num_bytes = ac->cardinality * sizeof(uint16_t); - memcpy(array_zone, ac->array, num_bytes); - array_zone += ac->cardinality; - count = ac->cardinality - 1; - break; + if (container2 != container1) { + leaf1->container = container2; + leaf1->typecode = typecode2; + } + + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); } - default: - roaring_unreachable; } - memcpy(&count_zone[i], &count, 2); + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); + } } - memcpy(key_zone, ra->keys, ra->size * sizeof(uint16_t)); - memcpy(typecode_zone, ra->typecodes, ra->size * sizeof(uint8_t)); - uint32_t header = ((uint32_t)ra->size << 15) | FROZEN_COOKIE; - memcpy(header_zone, &header, 4); } -const roaring_bitmap_t * -roaring_bitmap_frozen_view(const char *buf, size_t length) { - if ((uintptr_t)buf % 32 != 0) { - return NULL; +/** + * Flips the leaf at high48 in the range [min, max), returning a new leaf with a + * new container. If the high48 key is not found in the existing bitmap, a new + * container is created. Returns null if the negation results in an empty range. + */ +static leaf_t *roaring64_flip_leaf(const roaring64_bitmap_t *r, + uint8_t high48[], uint32_t min, + uint32_t max) { + leaf_t *leaf1 = (leaf_t *)art_find(&r->art, high48); + container_t *container2; + uint8_t typecode2; + if (leaf1 == NULL) { + // No container at this key, create a full container. + container2 = container_range_of_ones(min, max, &typecode2); + } else if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = + container_not(leaf1->container, leaf1->typecode, &typecode2); + } else { + // Partially flip a container. + container2 = container_not_range(leaf1->container, leaf1->typecode, min, + max, &typecode2); + } + if (container_nonzero_cardinality(container2, typecode2)) { + return create_leaf(container2, typecode2); } + container_free(container2, typecode2); + return NULL; +} - // cookie and num_containers - if (length < 4) { - return NULL; +/** + * Flips the leaf at high48 in the range [min, max). If the high48 key is not + * found in the bitmap, a new container is created. Deletes the leaf and + * associated container if the negation results in an empty range. + */ +static void roaring64_flip_leaf_inplace(roaring64_bitmap_t *r, uint8_t high48[], + uint32_t min, uint32_t max) { + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + container_t *container2; + uint8_t typecode2; + if (leaf == NULL) { + // No container at this key, insert a full container. + container2 = container_range_of_ones(min, max, &typecode2); + art_insert(&r->art, high48, + (art_val_t *)create_leaf(container2, typecode2)); + return; } - uint32_t header; - memcpy(&header, buf + length - 4, 4); // header may be misaligned - if ((header & 0x7FFF) != FROZEN_COOKIE) { - return NULL; + + if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = + container_inot(leaf->container, leaf->typecode, &typecode2); + } else { + // Partially flip a container. + container2 = container_inot_range(leaf->container, leaf->typecode, min, + max, &typecode2); } - int32_t num_containers = (header >> 15); - // typecodes, counts and keys - if (length < 4 + (size_t)num_containers * (1 + 2 + 2)) { - return NULL; + leaf->container = container2; + leaf->typecode = typecode2; + + if (!container_nonzero_cardinality(leaf->container, leaf->typecode)) { + art_erase(&r->art, high48); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); } - uint16_t *keys = (uint16_t *)(buf + length - 4 - num_containers * 5); - uint16_t *counts = (uint16_t *)(buf + length - 4 - num_containers * 3); - uint8_t *typecodes = (uint8_t *)(buf + length - 4 - num_containers * 1); +} - // {bitset,array,run}_zone - int32_t num_bitset_containers = 0; - int32_t num_run_containers = 0; - int32_t num_array_containers = 0; - size_t bitset_zone_size = 0; - size_t run_zone_size = 0; - size_t array_zone_size = 0; - for (int32_t i = 0; i < num_containers; i++) { - switch (typecodes[i]) { - case BITSET_CONTAINER_TYPE: - num_bitset_containers++; - bitset_zone_size += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; - case RUN_CONTAINER_TYPE: - num_run_containers++; - run_zone_size += counts[i] * sizeof(rle16_t); - break; - case ARRAY_CONTAINER_TYPE: - num_array_containers++; - array_zone_size += (counts[i] + UINT32_C(1)) * sizeof(uint16_t); - break; - default: - return NULL; +roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return roaring64_bitmap_copy(r); + } + return roaring64_bitmap_flip_closed(r, min, max - 1); +} + +roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r1, + uint64_t min, uint64_t max) { + if (min > max) { + return roaring64_bitmap_copy(r1); + } + uint8_t min_high48_key[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48_key); + uint8_t max_high48_key[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48_key); + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; + + roaring64_bitmap_t *r2 = roaring64_bitmap_create(); + art_iterator_t it = art_init_iterator(&r1->art, /*first=*/true); + + // Copy the containers before min unchanged. + while (it.value != NULL && compare_high48(it.key, min_high48_key) < 0) { + leaf_t *leaf1 = (leaf_t *)it.value; + uint8_t typecode2 = leaf1->typecode; + container_t *container2 = get_copy_of_container( + leaf1->container, &typecode2, /*copy_on_write=*/false); + art_insert(&r2->art, it.key, + (art_val_t *)create_leaf(container2, typecode2)); + art_iterator_next(&it); + } + + // Flip the range (including non-existent containers!) between min and max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); + + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; + } + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. + } + + leaf_t *leaf = roaring64_flip_leaf(r1, current_high48_key, + min_container, max_container); + if (leaf != NULL) { + art_insert(&r2->art, current_high48_key, (art_val_t *)leaf); + } + } + + // Copy the containers after max unchanged. + it = art_upper_bound(&r1->art, max_high48_key); + while (it.value != NULL) { + leaf_t *leaf1 = (leaf_t *)it.value; + uint8_t typecode2 = leaf1->typecode; + container_t *container2 = get_copy_of_container( + leaf1->container, &typecode2, /*copy_on_write=*/false); + art_insert(&r2->art, it.key, + (art_val_t *)create_leaf(container2, typecode2)); + art_iterator_next(&it); + } + + return r2; +} + +void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_flip_closed_inplace(r, min, max - 1); +} + +void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } + uint16_t min_low16 = (uint16_t)min; + uint16_t max_low16 = (uint16_t)max; + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; + + // Flip the range (including non-existent containers!) between min and max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); + + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; + } + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. + } + + roaring64_flip_leaf_inplace(r, current_high48_key, min_container, + max_container); + } +} + +// Returns the number of distinct high 32-bit entries in the bitmap. +static inline uint64_t count_high32(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t high32_count = 0; + uint32_t prev_high32 = 0; + while (it.value != NULL) { + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (high32_count == 0 || prev_high32 != current_high32) { + high32_count++; + prev_high32 = current_high32; + } + art_iterator_next(&it); + } + return high32_count; +} + +// Frees the (32-bit!) bitmap without freeing the containers. +static inline void roaring_bitmap_free_without_containers(roaring_bitmap_t *r) { + ra_clear_without_containers(&r->high_low_container); + roaring_free(r); +} + +size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + size_t size = 0; + + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count; + size += sizeof(high32_count); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint32_t prev_high32 = 0; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. + while (it.value != NULL) { + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + size += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); + } + + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && (uint32_t)(combine_key(it2.key, 0) >> + 32) == current_high32) { + containers_with_high32++; + art_iterator_next(&it2); + } + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; } + leaf_t *leaf = (leaf_t *)it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high32 >> 16), leaf->container, + leaf->typecode); + art_iterator_next(&it); } - if (length != bitset_zone_size + run_zone_size + array_zone_size + - 5 * num_containers + 4) { - return NULL; - } - uint64_t *bitset_zone = (uint64_t*) (buf); - rle16_t *run_zone = (rle16_t*) (buf + bitset_zone_size); - uint16_t *array_zone = (uint16_t*) (buf + bitset_zone_size + run_zone_size); - size_t alloc_size = 0; - alloc_size += sizeof(roaring_bitmap_t); - alloc_size += num_containers * sizeof(container_t*); - alloc_size += num_bitset_containers * sizeof(bitset_container_t); - alloc_size += num_run_containers * sizeof(run_container_t); - alloc_size += num_array_containers * sizeof(array_container_t); + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + size += sizeof(prev_high32); - char *arena = (char *)roaring_malloc(alloc_size); - if (arena == NULL) { - return NULL; + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); } - roaring_bitmap_t *rb = (roaring_bitmap_t *) - arena_alloc(&arena, sizeof(roaring_bitmap_t)); - rb->high_low_container.flags = ROARING_FLAG_FROZEN; - rb->high_low_container.allocation_size = num_containers; - rb->high_low_container.size = num_containers; - rb->high_low_container.keys = (uint16_t *)keys; - rb->high_low_container.typecodes = (uint8_t *)typecodes; - rb->high_low_container.containers = - (container_t **)arena_alloc(&arena, - sizeof(container_t*) * num_containers); - // Ensure offset of high_low_container.containers is known distance used in - // C++ wrapper. sizeof(roaring_bitmap_t) is used as it is the size of the - // only allocation that precedes high_low_container.containers. If this is - // changed (new allocation or changed order), this offset will also need to - // be changed in the C++ wrapper. - assert(rb == - (roaring_bitmap_t *)((char *)rb->high_low_container.containers - - sizeof(roaring_bitmap_t))); - for (int32_t i = 0; i < num_containers; i++) { - switch (typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - bitset_container_t *bitset = (bitset_container_t *) - arena_alloc(&arena, sizeof(bitset_container_t)); - bitset->words = bitset_zone; - bitset->cardinality = counts[i] + UINT32_C(1); - rb->high_low_container.containers[i] = bitset; - bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; - break; - } - case RUN_CONTAINER_TYPE: { - run_container_t *run = (run_container_t *) - arena_alloc(&arena, sizeof(run_container_t)); - run->capacity = counts[i]; - run->n_runs = counts[i]; - run->runs = run_zone; - rb->high_low_container.containers[i] = run; - run_zone += run->n_runs; - break; + return size; +} + +size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, + char *buf) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; + } + const char *initial_buf = buf; + + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count = count_high32(r); + memcpy(buf, &high32_count, sizeof(high32_count)); + buf += sizeof(high32_count); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint32_t prev_high32 = 0; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. + while (it.value != NULL) { + uint64_t current_high48 = combine_key(it.key, 0); + uint32_t current_high32 = (uint32_t)(current_high48 >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + memcpy(buf, &prev_high32, sizeof(prev_high32)); + buf += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); } - case ARRAY_CONTAINER_TYPE: { - array_container_t *array = (array_container_t *) - arena_alloc(&arena, sizeof(array_container_t)); - array->capacity = counts[i] + UINT32_C(1); - array->cardinality = counts[i] + UINT32_C(1); - array->array = array_zone; - rb->high_low_container.containers[i] = array; - array_zone += counts[i] + UINT32_C(1); - break; + + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && + (uint32_t)combine_key(it2.key, 0) == current_high32) { + containers_with_high32++; + art_iterator_next(&it2); } - default: - roaring_free(arena); - return NULL; + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; } + leaf_t *leaf = (leaf_t *)it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high48 >> 16), leaf->container, + leaf->typecode); + art_iterator_next(&it); } - return rb; + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + memcpy(buf, &prev_high32, sizeof(prev_high32)); + buf += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); + } + + return buf - initial_buf; } -ALLOW_UNALIGNED -roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { - char *start_of_buf = (char *) buf; - uint32_t cookie; - int32_t num_containers; - uint16_t *descriptive_headers; - uint32_t *offset_headers = NULL; - const char *run_flag_bitset = NULL; - bool hasrun = false; +size_t roaring64_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; + } + size_t read_bytes = 0; - // deserialize cookie - memcpy(&cookie, buf, sizeof(uint32_t)); - buf += sizeof(uint32_t); - if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { - memcpy(&num_containers, buf, sizeof(int32_t)); - buf += sizeof(int32_t); - descriptive_headers = (uint16_t *) buf; - buf += num_containers * 2 * sizeof(uint16_t); - offset_headers = (uint32_t *) buf; - buf += num_containers * sizeof(uint32_t); - } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { - num_containers = (cookie >> 16) + 1; - hasrun = true; - int32_t run_flag_bitset_size = (num_containers + 7) / 8; - run_flag_bitset = buf; - buf += run_flag_bitset_size; - descriptive_headers = (uint16_t *) buf; - buf += num_containers * 2 * sizeof(uint16_t); - if(num_containers >= NO_OFFSET_THRESHOLD) { - offset_headers = (uint32_t *) buf; - buf += num_containers * sizeof(uint32_t); - } - } else { - return NULL; + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { + return 0; } + memcpy(&buckets, buf, sizeof(buckets)); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); - // calculate total size for allocation - int32_t num_bitset_containers = 0; - int32_t num_run_containers = 0; - int32_t num_array_containers = 0; + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return 0; + } - for (int32_t i = 0; i < num_containers; i++) { - uint16_t tmp; - memcpy(&tmp, descriptive_headers + 2*i+1, sizeof(tmp)); - uint32_t cardinality = tmp + 1; - bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); - bool isrun = false; - if(hasrun) { - if((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { - isbitmap = false; - isrun = true; - } + // Iterate through buckets ordered by increasing keys. + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + return 0; } + buf += sizeof(high32); + read_bytes += sizeof(high32); - if (isbitmap) { - num_bitset_containers++; - } else if (isrun) { - num_run_containers++; - } else { - num_array_containers++; + // Read the 32-bit Roaring bitmaps representing the least significant + // bits of a set of elements. + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + buf, maxbytes - read_bytes); + if (bitmap32_size == 0) { + return 0; } + buf += bitmap32_size; + read_bytes += bitmap32_size; } + return read_bytes; +} - size_t alloc_size = 0; - alloc_size += sizeof(roaring_bitmap_t); - alloc_size += num_containers * sizeof(container_t*); - alloc_size += num_bitset_containers * sizeof(bitset_container_t); - alloc_size += num_run_containers * sizeof(run_container_t); - alloc_size += num_array_containers * sizeof(array_container_t); - alloc_size += num_containers * sizeof(uint16_t); // keys - alloc_size += num_containers * sizeof(uint8_t); // typecodes +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( + const char *buf, size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return NULL; + } + size_t read_bytes = 0; - // allocate bitmap and construct containers - char *arena = (char *)roaring_malloc(alloc_size); - if (arena == NULL) { + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { return NULL; } + memcpy(&buckets, buf, sizeof(buckets)); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); - roaring_bitmap_t *rb = (roaring_bitmap_t *) - arena_alloc(&arena, sizeof(roaring_bitmap_t)); - rb->high_low_container.flags = ROARING_FLAG_FROZEN; - rb->high_low_container.allocation_size = num_containers; - rb->high_low_container.size = num_containers; - rb->high_low_container.containers = - (container_t **)arena_alloc(&arena, - sizeof(container_t*) * num_containers); + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return NULL; + } - uint16_t *keys = (uint16_t *)arena_alloc(&arena, num_containers * sizeof(uint16_t)); - uint8_t *typecodes = (uint8_t *)arena_alloc(&arena, num_containers * sizeof(uint8_t)); + roaring64_bitmap_t *r = roaring64_bitmap_create(); + // Iterate through buckets ordered by increasing keys. + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + roaring64_bitmap_free(r); + return NULL; + } + memcpy(&high32, buf, sizeof(high32)); + buf += sizeof(high32); + read_bytes += sizeof(high32); - rb->high_low_container.keys = keys; - rb->high_low_container.typecodes = typecodes; + // Read the 32-bit Roaring bitmaps representing the least significant + // bits of a set of elements. + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + buf, maxbytes - read_bytes); + if (bitmap32_size == 0) { + roaring64_bitmap_free(r); + return NULL; + } - for (int32_t i = 0; i < num_containers; i++) { - uint16_t tmp; - memcpy(&tmp, descriptive_headers + 2*i+1, sizeof(tmp)); - int32_t cardinality = tmp + 1; - bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); - bool isrun = false; - if(hasrun) { - if((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { - isbitmap = false; - isrun = true; - } + roaring_bitmap_t *bitmap32 = roaring_bitmap_portable_deserialize_safe( + buf, maxbytes - read_bytes); + if (bitmap32 == NULL) { + roaring64_bitmap_free(r); + return NULL; } + buf += bitmap32_size; + read_bytes += bitmap32_size; + + // Insert all containers of the 32-bit bitmap into the 64-bit bitmap. + uint32_t r32_size = ra_get_size(&bitmap32->high_low_container); + for (size_t i = 0; i < r32_size; ++i) { + uint16_t key16 = + ra_get_key_at_index(&bitmap32->high_low_container, (uint16_t)i); + uint8_t typecode; + container_t *container = ra_get_container_at_index( + &bitmap32->high_low_container, (uint16_t)i, &typecode); + + uint64_t high48_bits = + (((uint64_t)high32) << 32) | (((uint64_t)key16) << 16); + uint8_t high48[ART_KEY_BYTES]; + split_key(high48_bits, high48); + leaf_t *leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); + } + roaring_bitmap_free_without_containers(bitmap32); + } + return r; +} + +bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, + roaring_iterator64 iterator, void *ptr) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + uint64_t high48 = combine_key(it.key, 0); + uint64_t high32 = high48 & 0xFFFFFFFF00000000ULL; + uint32_t low32 = high48; + leaf_t *leaf = (leaf_t *)it.value; + if (!container_iterate64(leaf->container, leaf->typecode, low32, + iterator, high32, ptr)) { + return false; + } + art_iterator_next(&it); + } + return true; +} - keys[i] = descriptive_headers[2*i]; +void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, + uint64_t *out) { + roaring64_iterator_t it = {0}; + roaring64_iterator_init_at(r, &it, /*first=*/true); + roaring64_iterator_read(&it, out, UINT64_MAX); +} - if (isbitmap) { - typecodes[i] = BITSET_CONTAINER_TYPE; - bitset_container_t *c = (bitset_container_t *)arena_alloc(&arena, sizeof(bitset_container_t)); - c->cardinality = cardinality; - if(offset_headers != NULL) { - c->words = (uint64_t *) (start_of_buf + offset_headers[i]); - } else { - c->words = (uint64_t *) buf; - buf += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - } - rb->high_low_container.containers[i] = c; - } else if (isrun) { - typecodes[i] = RUN_CONTAINER_TYPE; - run_container_t *c = (run_container_t *)arena_alloc(&arena, sizeof(run_container_t)); - c->capacity = cardinality; - uint16_t n_runs; - if(offset_headers != NULL) { - memcpy(&n_runs, start_of_buf + offset_headers[i], sizeof(uint16_t)); - c->n_runs = n_runs; - c->runs = (rle16_t *) (start_of_buf + offset_headers[i] + sizeof(uint16_t)); - } else { - memcpy(&n_runs, buf, sizeof(uint16_t)); - c->n_runs = n_runs; - buf += sizeof(uint16_t); - c->runs = (rle16_t *) buf; - buf += c->n_runs * sizeof(rle16_t); - } - rb->high_low_container.containers[i] = c; - } else { - typecodes[i] = ARRAY_CONTAINER_TYPE; - array_container_t *c = (array_container_t *)arena_alloc(&arena, sizeof(array_container_t)); - c->cardinality = cardinality; - c->capacity = cardinality; - if(offset_headers != NULL) { - c->array = (uint16_t *) (start_of_buf + offset_headers[i]); - } else { - c->array = (uint16_t *) buf; - buf += cardinality * sizeof(uint16_t); - } - rb->high_low_container.containers[i] = c; +roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/true); +} + +roaring64_iterator_t *roaring64_iterator_create_last( + const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/false); +} + +void roaring64_iterator_reinit(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/true); +} + +void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/false); +} + +roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { + roaring64_iterator_t *new_it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + memcpy(new_it, it, sizeof(*it)); + return new_it; +} + +void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } + +bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { + return it->has_value; +} + +uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { + return it->value; +} + +bool roaring64_iterator_advance(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (it->saturated_forward) { + return (it->has_value = false); } + roaring64_iterator_init_at(it->parent, it, /*first=*/true); + return it->has_value; } - - return rb; + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_next(leaf->container, leaf->typecode, + &it->container_it, &low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + if (art_iterator_next(&it->art_it)) { + return roaring64_iterator_init_at_leaf_first(it); + } + it->saturated_forward = true; + return (it->has_value = false); } -bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t * bitset) { - uint32_t max_value = roaring_bitmap_maximum(r); - size_t new_array_size = (size_t)(((uint64_t)max_value + 63)/64); - bool resize_ok = bitset_resize(bitset, new_array_size, true); - if(!resize_ok) { return false; } - const roaring_array_t *ra = &r->high_low_container; - for (int i = 0; i < ra->size; ++i) { - uint64_t* words = bitset->array + (ra->keys[i]<<10); - uint8_t type = ra->typecodes[i]; - const container_t *c = ra->containers[i]; - if(type == SHARED_CONTAINER_TYPE) { - c = container_unwrap_shared(c, &type); +bool roaring64_iterator_previous(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (!it->saturated_forward) { + // Saturated backward. + return (it->has_value = false); } - switch (type) { - case BITSET_CONTAINER_TYPE: - { - size_t max_word_index = new_array_size - (ra->keys[i]<<10); - if(max_word_index > 1024) { max_word_index = 1024; } - const bitset_container_t *src = const_CAST_bitset(c); - memcpy(words, src->words, max_word_index * sizeof(uint64_t)); - } - break; - case ARRAY_CONTAINER_TYPE: - { - const array_container_t *src = const_CAST_array(c); - bitset_set_list(words, src->array, src->cardinality); - } - break; - case RUN_CONTAINER_TYPE: - { - const run_container_t *src = const_CAST_run(c); - for (int32_t rlepos = 0; rlepos < src->n_runs; ++rlepos) { - rle16_t rle = src->runs[rlepos]; - bitset_set_lenrange(words, rle.value, rle.length); - } - } - break; - default: - roaring_unreachable; + roaring64_iterator_init_at(it->parent, it, /*first=*/false); + return it->has_value; + } + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_prev(leaf->container, leaf->typecode, + &it->container_it, &low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + if (art_iterator_prev(&it->art_it)) { + return roaring64_iterator_init_at_leaf_last(it); + } + it->saturated_forward = false; // Saturated backward. + return (it->has_value = false); +} + +bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, + uint64_t val) { + uint8_t val_high48[ART_KEY_BYTES]; + uint16_t val_low16 = split_key(val, val_high48); + if (!it->has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { + // The ART iterator is before or after the high48 bits of `val` (or + // beyond the ART altogether), so we need to move to a leaf with a key + // equal or greater. + if (!art_iterator_lower_bound(&it->art_it, val_high48)) { + // Only smaller keys found. + it->saturated_forward = true; + return (it->has_value = false); + } + it->high48 = combine_key(it->art_it.key, 0); + // Fall through to the next if statement. + } + + if (it->high48 == (val & 0xFFFFFFFFFFFF0000)) { + // We're at equal high bits, check if a suitable value can be found in + // this container. + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_lower_bound(leaf->container, leaf->typecode, + &it->container_it, &low16, + val_low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + // Only smaller entries in this container, move to the next. + if (!art_iterator_next(&it->art_it)) { + it->saturated_forward = true; + return (it->has_value = false); + } + } + + // We're at a leaf with high bits greater than `val`, so the first entry in + // this container is our result. + return roaring64_iterator_init_at_leaf_first(it); +} + +uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, + uint64_t count) { + uint64_t consumed = 0; + while (it->has_value && consumed < count) { + uint32_t container_consumed; + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; + } + bool has_value = container_iterator_read_into_uint64( + leaf->container, leaf->typecode, &it->container_it, it->high48, buf, + container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->has_value = true; + it->value = it->high48 | low16; + assert(consumed == count); + return consumed; + } + it->has_value = art_iterator_next(&it->art_it); + if (it->has_value) { + roaring64_iterator_init_at_leaf_first(it); } } - return true; + return consumed; } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { +} // extern "C" +} // namespace roaring +} // namespace api #endif -/* end file src/roaring.c */ +/* end file src/roaring64.c */ /* begin file src/roaring_array.c */ #include +#include #include #include #include #include -#include #ifdef __cplusplus -extern "C" { namespace roaring { namespace internal { +extern "C" { +namespace roaring { +namespace internal { #endif // Convention: [0,ra->size) all elements are initialized @@ -20071,20 +24763,22 @@ extern "C" { namespace roaring { namespace internal { extern inline int32_t ra_get_size(const roaring_array_t *ra); extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); -extern inline container_t *ra_get_container_at_index( - const roaring_array_t *ra, uint16_t i, - uint8_t *typecode); +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); extern inline void ra_unshare_container_at_index(roaring_array_t *ra, uint16_t i); -extern inline void ra_replace_key_and_container_at_index( - roaring_array_t *ra, int32_t i, uint16_t key, - container_t *c, uint8_t typecode); +extern inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, + int32_t i, + uint16_t key, + container_t *c, + uint8_t typecode); -extern inline void ra_set_container_at_index( - const roaring_array_t *ra, int32_t i, - container_t *c, uint8_t typecode); +extern inline void ra_set_container_at_index(const roaring_array_t *ra, + int32_t i, container_t *c, + uint8_t typecode); static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { // @@ -20092,16 +24786,17 @@ static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { // Struct-of-Arrays vs. Array-of-Structs: // https://github.com/RoaringBitmap/CRoaring/issues/256 - if ( new_capacity == 0 ) { - roaring_free(ra->containers); - ra->containers = NULL; - ra->keys = NULL; - ra->typecodes = NULL; - ra->allocation_size = 0; - return true; + if (new_capacity == 0) { + roaring_free(ra->containers); + ra->containers = NULL; + ra->keys = NULL; + ra->typecodes = NULL; + ra->allocation_size = 0; + return true; } - const size_t memoryneeded = new_capacity * ( - sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); + const size_t memoryneeded = + new_capacity * + (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); void *bigalloc = roaring_malloc(memoryneeded); if (!bigalloc) return false; void *oldbigalloc = ra->containers; @@ -20110,10 +24805,10 @@ static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { uint8_t *newtypecodes = (uint8_t *)(newkeys + new_capacity); assert((char *)(newtypecodes + new_capacity) == (char *)bigalloc + memoryneeded); - if(ra->size > 0) { - memcpy(newcontainers, ra->containers, sizeof(container_t *) * ra->size); - memcpy(newkeys, ra->keys, sizeof(uint16_t) * ra->size); - memcpy(newtypecodes, ra->typecodes, sizeof(uint8_t) * ra->size); + if (ra->size > 0) { + memcpy(newcontainers, ra->containers, sizeof(container_t *) * ra->size); + memcpy(newkeys, ra->keys, sizeof(uint16_t) * ra->size); + memcpy(newtypecodes, ra->typecodes, sizeof(uint8_t) * ra->size); } ra->containers = newcontainers; ra->keys = newkeys; @@ -20127,20 +24822,21 @@ bool ra_init_with_capacity(roaring_array_t *new_ra, uint32_t cap) { if (!new_ra) return false; ra_init(new_ra); - // Containers hold 64Ki elements, so 64Ki containers is enough to hold `0x10000 * 0x10000` (all 2^32) elements + // Containers hold 64Ki elements, so 64Ki containers is enough to hold + // `0x10000 * 0x10000` (all 2^32) elements if (cap > 0x10000) { cap = 0x10000; } - if(cap > 0) { - void *bigalloc = roaring_malloc(cap * - (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t))); - if( bigalloc == NULL ) return false; - new_ra->containers = (container_t **)bigalloc; - new_ra->keys = (uint16_t *)(new_ra->containers + cap); - new_ra->typecodes = (uint8_t *)(new_ra->keys + cap); - // Narrowing is safe because of above check - new_ra->allocation_size = (int32_t)cap; + if (cap > 0) { + void *bigalloc = roaring_malloc( + cap * (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t))); + if (bigalloc == NULL) return false; + new_ra->containers = (container_t **)bigalloc; + new_ra->keys = (uint16_t *)(new_ra->containers + cap); + new_ra->typecodes = (uint8_t *)(new_ra->keys + cap); + // Narrowing is safe because of above check + new_ra->allocation_size = (int32_t)cap; } return true; } @@ -20149,14 +24845,16 @@ int ra_shrink_to_fit(roaring_array_t *ra) { int savings = (ra->allocation_size - ra->size) * (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); if (!realloc_array(ra, ra->size)) { - return 0; + return 0; } ra->allocation_size = ra->size; return savings; } void ra_init(roaring_array_t *new_ra) { - if (!new_ra) { return; } + if (!new_ra) { + return; + } new_ra->keys = NULL; new_ra->containers = NULL; new_ra->typecodes = NULL; @@ -20169,9 +24867,9 @@ void ra_init(roaring_array_t *new_ra) { bool ra_overwrite(const roaring_array_t *source, roaring_array_t *dest, bool copy_on_write) { ra_clear_containers(dest); // we are going to overwrite them - if (source->size == 0) { // Note: can't call memcpy(NULL), even w/size - dest->size = 0; // <--- This is important. - return true; // output was just cleared, so they match + if (source->size == 0) { // Note: can't call memcpy(NULL), even w/size + dest->size = 0; // <--- This is important. + return true; // output was just cleared, so they match } if (dest->allocation_size < source->size) { if (!realloc_array(dest, source->size)) { @@ -20216,13 +24914,14 @@ void ra_clear_containers(roaring_array_t *ra) { } void ra_reset(roaring_array_t *ra) { - ra_clear_containers(ra); - ra->size = 0; - ra_shrink_to_fit(ra); + ra_clear_containers(ra); + ra->size = 0; + ra_shrink_to_fit(ra); } void ra_clear_without_containers(roaring_array_t *ra) { - roaring_free(ra->containers); // keys and typecodes are allocated with containers + roaring_free( + ra->containers); // keys and typecodes are allocated with containers ra->size = 0; ra->allocation_size = 0; ra->containers = NULL; @@ -20251,10 +24950,8 @@ bool extend_array(roaring_array_t *ra, int32_t k) { return true; } -void ra_append( - roaring_array_t *ra, uint16_t key, - container_t *c, uint8_t typecode -){ +void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, + uint8_t typecode) { extend_array(ra, 1); const int32_t pos = ra->size; @@ -20269,7 +24966,7 @@ void ra_append_copy(roaring_array_t *ra, const roaring_array_t *sa, extend_array(ra, 1); const int32_t pos = ra->size; - // old contents is junk not needing freeing + // old contents is junk that does not need freeing ra->keys[pos] = sa->keys[index]; // the shared container will be in two bitmaps if (copy_on_write) { @@ -20289,7 +24986,7 @@ void ra_append_copies_until(roaring_array_t *ra, const roaring_array_t *sa, uint16_t stopping_key, bool copy_on_write) { for (int32_t i = 0; i < sa->size; ++i) { if (sa->keys[i] >= stopping_key) break; - ra_append_copy(ra, sa, i, copy_on_write); + ra_append_copy(ra, sa, (uint16_t)i, copy_on_write); } } @@ -20360,18 +25057,17 @@ void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, } } -container_t *ra_get_container( - roaring_array_t *ra, uint16_t x, uint8_t *typecode -){ +container_t *ra_get_container(roaring_array_t *ra, uint16_t x, + uint8_t *typecode) { int i = binarySearch(ra->keys, (int32_t)ra->size, x); if (i < 0) return NULL; *typecode = ra->typecodes[i]; return ra->containers[i]; } -extern inline container_t *ra_get_container_at_index( - const roaring_array_t *ra, uint16_t i, - uint8_t *typecode); +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); extern inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, uint16_t i); @@ -20379,7 +25075,7 @@ extern inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); extern inline int32_t ra_advance_until(const roaring_array_t *ra, uint16_t x, - int32_t pos); + int32_t pos); // everything skipped over is freed int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos) { @@ -20390,10 +25086,8 @@ int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos) { return pos; } -void ra_insert_new_key_value_at( - roaring_array_t *ra, int32_t i, uint16_t key, - container_t *c, uint8_t typecode -){ +void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, + container_t *c, uint8_t typecode) { extend_array(ra, 1); // May be an optimization opportunity with DIY memmove memmove(&(ra->keys[i + 1]), &(ra->keys[i]), @@ -20462,8 +25156,7 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance) { } int32_t srcpos = ra->size - count; int32_t dstpos = srcpos + distance; - memmove(&(ra->keys[dstpos]), &(ra->keys[srcpos]), - sizeof(uint16_t) * count); + memmove(&(ra->keys[dstpos]), &(ra->keys[srcpos]), sizeof(uint16_t) * count); memmove(&(ra->containers[dstpos]), &(ra->containers[srcpos]), sizeof(container_t *) * count); memmove(&(ra->typecodes[dstpos]), &(ra->typecodes[srcpos]), @@ -20471,7 +25164,6 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance) { ra->size += distance; } - void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans) { size_t ctr = 0; for (int32_t i = 0; i < ra->size; ++i) { @@ -20482,7 +25174,8 @@ void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans) { } } -bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, size_t limit, uint32_t *ans) { +bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, + size_t limit, uint32_t *ans) { size_t ctr = 0; size_t dtr = 0; @@ -20495,9 +25188,8 @@ bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, size_t limi size_t cur_len = 0; for (int i = 0; i < ra->size; ++i) { - - const container_t *c = container_unwrap_shared( - ra->containers[i], &ra->typecodes[i]); + const container_t *c = + container_unwrap_shared(ra->containers[i], &ra->typecodes[i]); switch (ra->typecodes[i]) { case BITSET_CONTAINER_TYPE: t_limit = (const_CAST_bitset(c))->cardinality; @@ -20509,25 +25201,28 @@ bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, size_t limi t_limit = run_container_cardinality(const_CAST_run(c)); break; } - if (ctr + t_limit - 1 >= offset && ctr < offset + limit){ - if (!first){ - //first_skip = t_limit - (ctr + t_limit - offset); + if (ctr + t_limit - 1 >= offset && ctr < offset + limit) { + if (!first) { + // first_skip = t_limit - (ctr + t_limit - offset); first_skip = offset - ctr; first = true; - t_ans = (uint32_t *)roaring_malloc(sizeof(*t_ans) * (first_skip + limit)); - if(t_ans == NULL) { - return false; + t_ans = (uint32_t *)roaring_malloc(sizeof(*t_ans) * + (first_skip + limit)); + if (t_ans == NULL) { + return false; } - memset(t_ans, 0, sizeof(*t_ans) * (first_skip + limit)) ; + memset(t_ans, 0, sizeof(*t_ans) * (first_skip + limit)); cur_len = first_skip + limit; } - if (dtr + t_limit > cur_len){ - uint32_t * append_ans = (uint32_t *)roaring_malloc(sizeof(*append_ans) * (cur_len + t_limit)); - if(append_ans == NULL) { - if(t_ans != NULL) roaring_free(t_ans); - return false; + if (dtr + t_limit > cur_len) { + uint32_t *append_ans = (uint32_t *)roaring_malloc( + sizeof(*append_ans) * (cur_len + t_limit)); + if (append_ans == NULL) { + if (t_ans != NULL) roaring_free(t_ans); + return false; } - memset(append_ans, 0, sizeof(*append_ans) * (cur_len + t_limit)); + memset(append_ans, 0, + sizeof(*append_ans) * (cur_len + t_limit)); cur_len = cur_len + t_limit; memcpy(append_ans, t_ans, dtr * sizeof(uint32_t)); roaring_free(t_ans); @@ -20535,32 +25230,29 @@ bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, size_t limi } switch (ra->typecodes[i]) { case BITSET_CONTAINER_TYPE: - container_to_uint32_array( - t_ans + dtr, - const_CAST_bitset(c), ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); + container_to_uint32_array(t_ans + dtr, const_CAST_bitset(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); break; case ARRAY_CONTAINER_TYPE: - container_to_uint32_array( - t_ans + dtr, - const_CAST_array(c), ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); + container_to_uint32_array(t_ans + dtr, const_CAST_array(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); break; case RUN_CONTAINER_TYPE: - container_to_uint32_array( - t_ans + dtr, - const_CAST_run(c), ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); + container_to_uint32_array(t_ans + dtr, const_CAST_run(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); break; } dtr += t_limit; } ctr += t_limit; - if (dtr-first_skip >= limit) break; + if (dtr - first_skip >= limit) break; } - if(t_ans != NULL) { - memcpy(ans, t_ans+first_skip, limit * sizeof(uint32_t)); - free(t_ans); + if (t_ans != NULL) { + memcpy(ans, t_ans + first_skip, limit * sizeof(uint32_t)); + free(t_ans); } return true; } @@ -20602,7 +25294,7 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { uint32_t startOffset = 0; bool hasrun = ra_has_run_container(ra); if (hasrun) { - uint32_t cookie = SERIAL_COOKIE | ((ra->size - 1) << 16); + uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); memcpy(buf, &cookie, sizeof(cookie)); buf += sizeof(cookie); uint32_t s = (ra->size + 7) / 8; @@ -20637,8 +25329,9 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { buf += sizeof(ra->keys[k]); // get_cardinality returns a value in [1,1<<16], subtracting one // we get [0,1<<16 - 1] which fits in 16 bits - uint16_t card = (uint16_t)( - container_get_cardinality(ra->containers[k], ra->typecodes[k]) - 1); + uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], + ra->typecodes[k]) - + 1); memcpy(buf, &card, sizeof(card)); buf += sizeof(card); } @@ -20666,8 +25359,8 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { // Otherwise, it returns how many bytes are occupied. // size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { - size_t bytestotal = sizeof(int32_t);// for cookie - if(bytestotal > maxbytes) return 0; + size_t bytestotal = sizeof(int32_t); // for cookie + if (bytestotal > maxbytes) return 0; uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); buf += sizeof(uint32_t); @@ -20681,87 +25374,90 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { size = (cookie >> 16) + 1; else { bytestotal += sizeof(int32_t); - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; memcpy(&size, buf, sizeof(int32_t)); buf += sizeof(uint32_t); } - if (size > (1<<16)) { - return 0; + if (size > (1 << 16)) { + return 0; } char *bitmapOfRunContainers = NULL; bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; if (hasrun) { int32_t s = (size + 7) / 8; bytestotal += s; - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; bitmapOfRunContainers = (char *)buf; buf += s; } bytestotal += size * 2 * sizeof(uint16_t); - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; uint16_t *keyscards = (uint16_t *)buf; buf += size * 2 * sizeof(uint16_t); if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { // skipping the offsets bytestotal += size * 4; - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; buf += size * 4; } // Reading the containers for (int32_t k = 0; k < size; ++k) { uint16_t tmp; - memcpy(&tmp, keyscards + 2*k+1, sizeof(tmp)); + memcpy(&tmp, keyscards + 2 * k + 1, sizeof(tmp)); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; - if(hasrun) { - if((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { - isbitmap = false; - isrun = true; - } + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } } if (isbitmap) { - size_t containersize = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); bytestotal += containersize; - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; buf += containersize; } else if (isrun) { bytestotal += sizeof(uint16_t); - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); buf += sizeof(uint16_t); size_t containersize = n_runs * sizeof(rle16_t); bytestotal += containersize; - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; buf += containersize; } else { size_t containersize = thiscard * sizeof(uint16_t); bytestotal += containersize; - if(bytestotal > maxbytes) return 0; + if (bytestotal > maxbytes) return 0; buf += containersize; } } return bytestotal; } -// This function populates answer from the content of buf (reading up to maxbytes bytes). -// The function returns false if a properly serialized bitmap cannot be found. -// If it returns true, readbytes is populated by how many bytes were read, we have that *readbytes <= maxbytes. +// This function populates answer from the content of buf (reading up to +// maxbytes bytes). The function returns false if a properly serialized bitmap +// cannot be found. If it returns true, readbytes is populated by how many bytes +// were read, we have that *readbytes <= maxbytes. // // This function is endian-sensitive. -bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const size_t maxbytes, size_t * readbytes) { - *readbytes = sizeof(int32_t);// for cookie - if(*readbytes > maxbytes) { - // Ran out of bytes while reading first 4 bytes. - return false; +bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, + const size_t maxbytes, size_t *readbytes) { + *readbytes = sizeof(int32_t); // for cookie + if (*readbytes > maxbytes) { + // Ran out of bytes while reading first 4 bytes. + return false; } uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); buf += sizeof(uint32_t); if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { - // "I failed to find one of the right cookies. + // "I failed to find one of the right cookies. return false; } int32_t size; @@ -20770,29 +25466,30 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz size = (cookie >> 16) + 1; else { *readbytes += sizeof(int32_t); - if(*readbytes > maxbytes) { - // Ran out of bytes while reading second part of the cookie. - return false; + if (*readbytes > maxbytes) { + // Ran out of bytes while reading second part of the cookie. + return false; } memcpy(&size, buf, sizeof(int32_t)); buf += sizeof(uint32_t); } if (size < 0) { - // You cannot have a negative number of containers, the data must be corrupted. - return false; + // You cannot have a negative number of containers, the data must be + // corrupted. + return false; } - if (size > (1<<16)) { - // You cannot have so many containers, the data must be corrupted. - return false; + if (size > (1 << 16)) { + // You cannot have so many containers, the data must be corrupted. + return false; } const char *bitmapOfRunContainers = NULL; bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; if (hasrun) { int32_t s = (size + 7) / 8; *readbytes += s; - if(*readbytes > maxbytes) {// data is corrupted? - // Ran out of bytes while reading run bitmap. - return false; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading run bitmap. + return false; } bitmapOfRunContainers = buf; buf += s; @@ -20800,9 +25497,9 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz uint16_t *keyscards = (uint16_t *)buf; *readbytes += size * 2 * sizeof(uint16_t); - if(*readbytes > maxbytes) { - // Ran out of bytes while reading key-cardinality array. - return false; + if (*readbytes > maxbytes) { + // Ran out of bytes while reading key-cardinality array. + return false; } buf += size * 2 * sizeof(uint16_t); @@ -20814,15 +25511,16 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz for (int32_t k = 0; k < size; ++k) { uint16_t tmp; - memcpy(&tmp, keyscards + 2*k, sizeof(tmp)); + memcpy(&tmp, keyscards + 2 * k, sizeof(tmp)); answer->keys[k] = tmp; } if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { *readbytes += size * 4; - if(*readbytes > maxbytes) {// data is corrupted? - // Ran out of bytes while reading offsets. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading offsets. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } // skipping the offsets @@ -20831,31 +25529,34 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz // Reading the containers for (int32_t k = 0; k < size; ++k) { uint16_t tmp; - memcpy(&tmp, keyscards + 2*k+1, sizeof(tmp)); + memcpy(&tmp, keyscards + 2 * k + 1, sizeof(tmp)); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; - if(hasrun) { - if((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { - isbitmap = false; - isrun = true; - } + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } } if (isbitmap) { // we check that the read is allowed - size_t containersize = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); *readbytes += containersize; - if(*readbytes > maxbytes) { - // Running out of bytes while reading a bitset container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (*readbytes > maxbytes) { + // Running out of bytes while reading a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } // it is now safe to read bitset_container_t *c = bitset_container_create(); - if(c == NULL) {// memory allocation failure - // Failed to allocate memory for a bitset container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } answer->size++; buf += bitset_container_read(thiscard, c, buf); @@ -20864,27 +25565,30 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz } else if (isrun) { // we check that the read is allowed *readbytes += sizeof(uint16_t); - if(*readbytes > maxbytes) { - // Running out of bytes while reading a run container (header). - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (*readbytes > maxbytes) { + // Running out of bytes while reading a run container (header). + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); size_t containersize = n_runs * sizeof(rle16_t); *readbytes += containersize; - if(*readbytes > maxbytes) {// data is corrupted? - // Running out of bytes while reading a run container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } // it is now safe to read run_container_t *c = run_container_create(); - if(c == NULL) {// memory allocation failure - // Failed to allocate memory for a run container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } answer->size++; buf += run_container_read(thiscard, c, buf); @@ -20894,18 +25598,20 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz // we check that the read is allowed size_t containersize = thiscard * sizeof(uint16_t); *readbytes += containersize; - if(*readbytes > maxbytes) {// data is corrupted? - // Running out of bytes while reading an array container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } // it is now safe to read array_container_t *c = array_container_create_given_capacity(thiscard); - if(c == NULL) {// memory allocation failure - // Failed to allocate memory for an array container. - ra_clear(answer);// we need to clear the containers already allocated, and the roaring array - return false; + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; } answer->size++; buf += array_container_read(thiscard, c, buf); @@ -20917,16 +25623,19 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const siz } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace internal { +} +} +} // extern "C" { namespace roaring { namespace internal { #endif /* end file src/roaring_array.c */ /* begin file src/roaring_priority_queue.c */ - #ifdef __cplusplus using namespace ::roaring::internal; -extern "C" { namespace roaring { namespace api { +extern "C" { +namespace roaring { +namespace api { #endif struct roaring_pq_element_s { @@ -20961,9 +25670,7 @@ static void pq_add(roaring_pq_t *pq, roaring_pq_element_t *t) { pq->elements[i] = *t; } -static void pq_free(roaring_pq_t *pq) { - roaring_free(pq); -} +static void pq_free(roaring_pq_t *pq) { roaring_free(pq); } static void percolate_down(roaring_pq_t *pq, uint32_t i) { uint32_t size = (uint32_t)pq->size; @@ -20989,7 +25696,8 @@ static void percolate_down(roaring_pq_t *pq, uint32_t i) { } static roaring_pq_t *create_pq(const roaring_bitmap_t **arr, uint32_t length) { - size_t alloc_size = sizeof(roaring_pq_t) + sizeof(roaring_pq_element_t) * length; + size_t alloc_size = + sizeof(roaring_pq_t) + sizeof(roaring_pq_element_t) * length; roaring_pq_t *answer = (roaring_pq_t *)roaring_malloc(alloc_size); answer->elements = (roaring_pq_element_t *)(answer + 1); answer->size = length; @@ -21034,29 +25742,30 @@ static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); int pos1 = 0, pos2 = 0; uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); while (true) { if (s1 == s2) { // todo: unsharing can be inefficient as it may create a clone where // none // is needed, but it has the benefit of being easy to reason about. - ra_unshare_container_at_index(&x1->high_low_container, pos1); - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); + ra_unshare_container_at_index(&x1->high_low_container, + (uint16_t)pos1); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); assert(type1 != SHARED_CONTAINER_TYPE); - ra_unshare_container_at_index(&x2->high_low_container, pos2); - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); + ra_unshare_container_at_index(&x2->high_low_container, + (uint16_t)pos2); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); assert(type2 != SHARED_CONTAINER_TYPE); container_t *c; if ((type2 == BITSET_CONTAINER_TYPE) && - (type1 != BITSET_CONTAINER_TYPE) - ){ + (type1 != BITSET_CONTAINER_TYPE)) { c = container_lazy_ior(c2, type2, c1, type1, &result_type); container_free(c1, type1); if (c != c2) { @@ -21077,24 +25786,24 @@ static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, ++pos2; if (pos1 == length1) break; if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, pos1, &type1); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); ra_append(&answer->high_low_container, s1, c1, type1); pos1++; if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, pos1); + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, pos2, &type2); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); ra_append(&answer->high_low_container, s2, c2, type2); pos2++; if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, pos2); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } if (pos1 == length1) { @@ -21167,6 +25876,8 @@ roaring_bitmap_t *roaring_bitmap_or_many_heap(uint32_t number, } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace api { +} +} +} // extern "C" { namespace roaring { namespace api { #endif /* end file src/roaring_priority_queue.c */ diff --git a/croaring-sys/CRoaring/roaring.h b/croaring-sys/CRoaring/roaring.h index d9e219b..ffef052 100644 --- a/croaring-sys/CRoaring/roaring.h +++ b/croaring-sys/CRoaring/roaring.h @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2023-09-27T16:30:23Z +// Created by amalgamation.sh on 2024-04-02T13:42:32Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -55,17 +55,18 @@ */ /* begin file include/roaring/roaring_version.h */ -// /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand -#ifndef ROARING_INCLUDE_ROARING_VERSION -#define ROARING_INCLUDE_ROARING_VERSION -#define ROARING_VERSION "2.0.2" -enum { - ROARING_VERSION_MAJOR = 2, +// clang-format off +// /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand +#ifndef ROARING_INCLUDE_ROARING_VERSION +#define ROARING_INCLUDE_ROARING_VERSION +#define ROARING_VERSION "3.0.1" +enum { + ROARING_VERSION_MAJOR = 3, ROARING_VERSION_MINOR = 0, - ROARING_VERSION_REVISION = 2 -}; -#endif // ROARING_INCLUDE_ROARING_VERSION -/* end file include/roaring/roaring_version.h */ + ROARING_VERSION_REVISION = 1 +}; +#endif // ROARING_INCLUDE_ROARING_VERSION +// clang-format on/* end file include/roaring/roaring_version.h */ /* begin file include/roaring/roaring_types.h */ /* Typedefs used by various components @@ -78,10 +79,11 @@ enum { #include #ifdef __cplusplus -extern "C" { namespace roaring { namespace api { +extern "C" { +namespace roaring { +namespace api { #endif - /** * When building .c files as C++, there's added compile-time checking if the * container types are derived from a `container_t` base class. So long as @@ -95,12 +97,12 @@ extern "C" { namespace roaring { namespace api { * code #undefs that after declaring `typedef ROARING_CONTAINER_T container_t;` */ #if defined(__cplusplus) - extern "C++" { - struct container_s {}; - } - #define ROARING_CONTAINER_T ::roaring::api::container_s +extern "C++" { +struct container_s {}; +} +#define ROARING_CONTAINER_T ::roaring::api::container_s #else - #define ROARING_CONTAINER_T void // no compile-time checking +#define ROARING_CONTAINER_T void // no compile-time checking #endif #define ROARING_FLAG_COW UINT8_C(0x1) @@ -125,15 +127,14 @@ typedef struct roaring_array_s { uint8_t flags; } roaring_array_t; - typedef bool (*roaring_iterator)(uint32_t value, void *param); typedef bool (*roaring_iterator64)(uint64_t value, void *param); /** -* (For advanced users.) -* The roaring_statistics_t can be used to collect detailed statistics about -* the composition of a roaring bitmap. -*/ + * (For advanced users.) + * The roaring_statistics_t can be used to collect detailed statistics about + * the composition of a roaring bitmap. + */ typedef struct roaring_statistics_s { uint32_t n_containers; /* number of containers */ @@ -166,8 +167,19 @@ typedef struct roaring_statistics_s { // and n_values_arrays, n_values_rle, n_values_bitmap } roaring_statistics_t; +/** + * Roaring-internal type used to iterate within a roaring container. + */ +typedef struct roaring_container_iterator_s { + // For bitset and array containers this is the index of the bit / entry. + // For run containers this points at the run. + int32_t index; +} roaring_container_iterator_t; + #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace api { +} +} +} // extern "C" { namespace roaring { namespace api { #endif #endif /* ROARING_TYPES_H */ @@ -178,25 +190,25 @@ typedef struct roaring_statistics_s { * */ - /** - * All macros should be prefixed with either CROARING or ROARING. - * The library uses both ROARING_... - * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for - * macros that are provided by the build system or that are closely - * related to the format. The header macros may also use ROARING_. - * The CROARING_ prefix is for internal macros that a user is unlikely - * to ever interact with. - */ +/** + * All macros should be prefixed with either CROARING or ROARING. + * The library uses both ROARING_... + * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for + * macros that are provided by the build system or that are closely + * related to the format. The header macros may also use ROARING_. + * The CROARING_ prefix is for internal macros that a user is unlikely + * to ever interact with. + */ -#ifndef INCLUDE_PORTABILITY_H_ -#define INCLUDE_PORTABILITY_H_ +#ifndef CROARING_INCLUDE_PORTABILITY_H_ +#define CROARING_INCLUDE_PORTABILITY_H_ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS 1 -#endif // __STDC_FORMAT_MACROS +#endif // __STDC_FORMAT_MACROS #ifdef _MSC_VER #define CROARING_VISUAL_STUDIO 1 @@ -211,8 +223,8 @@ typedef struct roaring_statistics_s { #else // just regular visual studio (best guess) #define CROARING_REGULAR_VISUAL_STUDIO 1 -#endif // __clang__ -#endif // _MSC_VER +#endif // __clang__ +#endif // _MSC_VER #ifndef CROARING_VISUAL_STUDIO #define CROARING_VISUAL_STUDIO 0 #endif @@ -229,10 +241,10 @@ typedef struct roaring_statistics_s { #ifndef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200809L -#endif // !(defined(_POSIX_C_SOURCE)) || (_POSIX_C_SOURCE < 200809L) +#endif // !(defined(_POSIX_C_SOURCE)) || (_POSIX_C_SOURCE < 200809L) #if !(defined(_XOPEN_SOURCE)) || (_XOPEN_SOURCE < 700) #define _XOPEN_SOURCE 700 -#endif // !(defined(_XOPEN_SOURCE)) || (_XOPEN_SOURCE < 700) +#endif // !(defined(_XOPEN_SOURCE)) || (_XOPEN_SOURCE < 700) #ifdef __illumos__ #define __EXTENSIONS__ @@ -256,10 +268,8 @@ extern "C" { // portability definitions are in global scope, not a namespace #if CROARING_REGULAR_VISUAL_STUDIO #ifndef __restrict__ #define __restrict__ __restrict -#endif // __restrict__ -#endif // CROARING_REGULAR_VISUAL_STUDIO - - +#endif // __restrict__ +#endif // CROARING_REGULAR_VISUAL_STUDIO #if defined(__x86_64__) || defined(_M_X64) // we have an x64 processor @@ -270,7 +280,7 @@ extern "C" { // portability definitions are in global scope, not a namespace #undef CROARING_IS_X64 #endif -#if defined(__clang_major__) && (__clang_major__<= 8) && !defined(__AVX2__) +#if defined(__clang_major__) && (__clang_major__ <= 8) && !defined(__AVX2__) // Older versions of clang have a bug affecting us // https://stackoverflow.com/questions/57228537/how-does-one-use-pragma-clang-attribute-push-with-c-namespaces #undef CROARING_IS_X64 @@ -284,8 +294,6 @@ extern "C" { // portability definitions are in global scope, not a namespace /* Non-Microsoft C/C++-compatible compiler */ #include // on some recent GCC, this will declare posix_memalign - - #if CROARING_CLANG_VISUAL_STUDIO /** @@ -303,6 +311,8 @@ extern "C" { // portability definitions are in global scope, not a namespace * (or ) before, so the headers * are fooled. */ +// To avoid reordering imports: +// clang-format off #include // for _blsr_u64 #include // for __lzcnt64 #include // for most things (AVX2, AVX512, _popcnt64) @@ -321,24 +331,24 @@ extern "C" { // portability definitions are in global scope, not a namespace #include #include #include -#endif // _MSC_VER >= 1920 +// clang-format on +#endif // _MSC_VER >= 1920 // unfortunately, we may not get _blsr_u64, but, thankfully, clang // has it as a macro. #ifndef _blsr_u64 // we roll our own #define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -#endif // CROARING_REGULAR_VISUAL_STUDIO -#endif // defined(__x86_64__) || defined(_M_X64) +#endif // CROARING_REGULAR_VISUAL_STUDIO +#endif // defined(__x86_64__) || defined(_M_X64) #if !defined(CROARING_USENEON) && !defined(DISABLENEON) && defined(__ARM_NEON) -# define CROARING_USENEON +#define CROARING_USENEON #endif #if defined(CROARING_USENEON) -# include +#include #endif #if !CROARING_REGULAR_VISUAL_STUDIO @@ -357,51 +367,59 @@ extern "C" { // portability definitions are in global scope, not a namespace // sadly there is no way to check whether we are missing these intrinsics // specifically. -/* wrappers for Visual Studio built-ins that look like gcc built-ins __builtin_ctzll */ -/* result might be undefined when input_num is zero */ +/* wrappers for Visual Studio built-ins that look like gcc built-ins + * __builtin_ctzll */ +/** result might be undefined when input_num is zero */ inline int roaring_trailing_zeroes(unsigned long long input_num) { unsigned long index; #ifdef _WIN64 // highly recommended!!! _BitScanForward64(&index, input_num); -#else // if we must support 32-bit Windows +#else // if we must support 32-bit Windows if ((uint32_t)input_num != 0) { _BitScanForward(&index, (uint32_t)input_num); } else { _BitScanForward(&index, (uint32_t)(input_num >> 32)); index += 32; } -#endif // _WIN64 +#endif // _WIN64 return index; } -/* wrappers for Visual Studio built-ins that look like gcc built-ins __builtin_clzll */ -/* result might be undefined when input_num is zero */ +/* wrappers for Visual Studio built-ins that look like gcc built-ins + * __builtin_clzll */ +/** result might be undefined when input_num is zero */ inline int roaring_leading_zeroes(unsigned long long input_num) { unsigned long index; #ifdef _WIN64 // highly recommended!!! _BitScanReverse64(&index, input_num); -#else // if we must support 32-bit Windows +#else // if we must support 32-bit Windows if (input_num > 0xFFFFFFFF) { _BitScanReverse(&index, (uint32_t)(input_num >> 32)); index += 32; } else { _BitScanReverse(&index, (uint32_t)(input_num)); } -#endif // _WIN64 +#endif // _WIN64 return 63 - index; } /* Use #define so this is effective even under /Ob0 (no inline) */ #define roaring_unreachable __assume(0) -#endif // __clang__ +#endif // __clang__ -#endif // CROARING_REGULAR_VISUAL_STUDIO +#endif // CROARING_REGULAR_VISUAL_STUDIO #ifndef CROARING_INTRINSICS #define CROARING_INTRINSICS 1 #define roaring_unreachable __builtin_unreachable() -static inline int roaring_trailing_zeroes(unsigned long long input_num) { return __builtin_ctzll(input_num); } -static inline int roaring_leading_zeroes(unsigned long long input_num) { return __builtin_clzll(input_num); } +/** result might be undefined when input_num is zero */ +inline int roaring_trailing_zeroes(unsigned long long input_num) { + return __builtin_ctzll(input_num); +} +/** result might be undefined when input_num is zero */ +inline int roaring_leading_zeroes(unsigned long long input_num) { + return __builtin_clzll(input_num); +} #endif #if CROARING_REGULAR_VISUAL_STUDIO @@ -414,46 +432,52 @@ static inline int roaring_leading_zeroes(unsigned long long input_num) { return #endif #if defined(__GNUC__) || defined(__clang__) -#define WARN_UNUSED __attribute__((warn_unused_result)) +#define CROARING_WARN_UNUSED __attribute__((warn_unused_result)) #else -#define WARN_UNUSED +#define CROARING_WARN_UNUSED #endif #define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100) #ifdef CROARING_USENEON // we can always compute the popcount fast. -#elif (defined(_M_ARM) || defined(_M_ARM64)) && ((defined(_WIN64) || defined(_WIN32)) && defined(CROARING_REGULAR_VISUAL_STUDIO) && CROARING_REGULAR_VISUAL_STUDIO) +#elif (defined(_M_ARM) || defined(_M_ARM64)) && \ + ((defined(_WIN64) || defined(_WIN32)) && \ + defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO) // we will need this function: static inline int roaring_hamming_backup(uint64_t x) { - uint64_t c1 = UINT64_C(0x5555555555555555); - uint64_t c2 = UINT64_C(0x3333333333333333); - uint64_t c4 = UINT64_C(0x0F0F0F0F0F0F0F0F); - x -= (x >> 1) & c1; - x = (( x >> 2) & c2) + (x & c2); x=(x +(x>>4))&c4; - x *= UINT64_C(0x0101010101010101); - return x >> 56; + uint64_t c1 = UINT64_C(0x5555555555555555); + uint64_t c2 = UINT64_C(0x3333333333333333); + uint64_t c4 = UINT64_C(0x0F0F0F0F0F0F0F0F); + x -= (x >> 1) & c1; + x = ((x >> 2) & c2) + (x & c2); + x = (x + (x >> 4)) & c4; + x *= UINT64_C(0x0101010101010101); + return x >> 56; } #endif - static inline int roaring_hamming(uint64_t x) { -#if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && CROARING_REGULAR_VISUAL_STUDIO +#if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO #ifdef CROARING_USENEON - return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); + return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); #elif defined(_M_ARM64) - return roaring_hamming_backup(x); - // (int) _CountOneBits64(x); is unavailable -#else // _M_ARM64 - return (int) __popcnt64(x); -#endif // _M_ARM64 -#elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && CROARING_REGULAR_VISUAL_STUDIO + return roaring_hamming_backup(x); + // (int) _CountOneBits64(x); is unavailable +#else // _M_ARM64 + return (int)__popcnt64(x); +#endif // _M_ARM64 +#elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO #ifdef _M_ARM - return roaring_hamming_backup(x); - // _CountOneBits is unavailable -#else // _M_ARM - return (int) __popcnt(( unsigned int)x) + (int) __popcnt(( unsigned int)(x>>32)); -#endif // _M_ARM + return roaring_hamming_backup(x); + // _CountOneBits is unavailable +#else // _M_ARM + return (int)__popcnt((unsigned int)x) + + (int)__popcnt((unsigned int)(x >> 32)); +#endif // _M_ARM #else return __builtin_popcountll(x); #endif @@ -461,16 +485,15 @@ static inline int roaring_hamming(uint64_t x) { #ifndef UINT64_C #define UINT64_C(c) (c##ULL) -#endif // UINT64_C +#endif // UINT64_C #ifndef UINT32_C #define UINT32_C(c) (c##UL) -#endif // UINT32_C +#endif // UINT32_C #ifdef __cplusplus } // extern "C" { -#endif // __cplusplus - +#endif // __cplusplus // this is almost standard? #undef STRINGIFY_IMPLEMENTATION_ @@ -489,7 +512,8 @@ static inline int roaring_hamming(uint64_t x) { // slower, but it should run everywhere. // -// Enable valid runtime implementations, and select CROARING_BUILTIN_IMPLEMENTATION +// Enable valid runtime implementations, and select +// CROARING_BUILTIN_IMPLEMENTATION // // We are going to use runtime dispatch. @@ -497,20 +521,20 @@ static inline int roaring_hamming(uint64_t x) { #ifdef __clang__ // clang does not have GCC push pop // warning: clang attribute push can't be used within a namespace in clang up -// til 8.0 so CROARING_TARGET_REGION and CROARING_UNTARGET_REGION must be *outside* of a -// namespace. -#define CROARING_TARGET_REGION(T) \ - _Pragma(STRINGIFY( \ - clang attribute push(__attribute__((target(T))), apply_to = function))) +// til 8.0 so CROARING_TARGET_REGION and CROARING_UNTARGET_REGION must be +// *outside* of a namespace. +#define CROARING_TARGET_REGION(T) \ + _Pragma(STRINGIFY(clang attribute push(__attribute__((target(T))), \ + apply_to = function))) #define CROARING_UNTARGET_REGION _Pragma("clang attribute pop") #elif defined(__GNUC__) // GCC is easier -#define CROARING_TARGET_REGION(T) \ - _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) +#define CROARING_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) #define CROARING_UNTARGET_REGION _Pragma("GCC pop_options") -#endif // clang then gcc +#endif // clang then gcc -#endif // CROARING_IS_X64 +#endif // CROARING_IS_X64 // Default target region macros don't do anything. #ifndef CROARING_TARGET_REGION @@ -518,9 +542,12 @@ static inline int roaring_hamming(uint64_t x) { #define CROARING_UNTARGET_REGION #endif - -#define CROARING_TARGET_AVX2 CROARING_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#define CROARING_TARGET_AVX512 CROARING_TARGET_REGION("avx2,bmi,bmi2,pclmul,lzcnt,popcnt,avx512f,avx512dq,avx512bw,avx512vbmi2,avx512bitalg,avx512vpopcntdq") +#define CROARING_TARGET_AVX2 \ + CROARING_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") +#define CROARING_TARGET_AVX512 \ + CROARING_TARGET_REGION( \ + "avx2,bmi,bmi2,pclmul,lzcnt,popcnt,avx512f,avx512dq,avx512bw," \ + "avx512vbmi2,avx512bitalg,avx512vpopcntdq") #define CROARING_UNTARGET_AVX2 CROARING_UNTARGET_REGION #define CROARING_UNTARGET_AVX512 CROARING_UNTARGET_REGION @@ -533,7 +560,9 @@ static inline int roaring_hamming(uint64_t x) { #define CROARING_UNTARGET_AVX2 #endif -#if defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && defined(__AVX512VBMI2__) && defined(__AVX512BITALG__) && defined(__AVX512VPOPCNTDQ__) +#if defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && \ + defined(__AVX512VBMI2__) && defined(__AVX512BITALG__) && \ + defined(__AVX512VPOPCNTDQ__) // No need for runtime dispatching. // It is unnecessary and harmful to old clang to tag regions. #undef CROARING_TARGET_AVX512 @@ -550,61 +579,101 @@ static inline int roaring_hamming(uint64_t x) { #endif #if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) - #define CROARING_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#define CROARING_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #elif defined(_WIN32) - #define CROARING_IS_BIG_ENDIAN 0 - #else - #if defined(__APPLE__) || defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ - #include - #elif defined(sun) || defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) - #include - #else // defined(__APPLE__) || defined(__FreeBSD__) - - #ifdef __has_include - #if __has_include() - #include - #endif //__has_include() - #endif //__has_include - - #endif // defined(__APPLE__) || defined(__FreeBSD__) - - - #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) - #define CROARING_IS_BIG_ENDIAN 0 - #endif - - #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - #define CROARING_IS_BIG_ENDIAN 0 - #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - #define CROARING_IS_BIG_ENDIAN 1 - #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define CROARING_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ +#include +#elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) +#include +#else // defined(__APPLE__) || defined(__FreeBSD__) + +#ifdef __has_include +#if __has_include() +#include +#endif //__has_include() +#endif //__has_include + +#endif // defined(__APPLE__) || defined(__FreeBSD__) + +#ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) +#define CROARING_IS_BIG_ENDIAN 0 #endif +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define CROARING_IS_BIG_ENDIAN 0 +#else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define CROARING_IS_BIG_ENDIAN 1 +#endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#endif + +// Host <-> big endian conversion. +#if CROARING_IS_BIG_ENDIAN +#define croaring_htobe64(x) (x) + +#elif defined(_WIN32) || defined(_WIN64) // CROARING_IS_BIG_ENDIAN +#include +#define croaring_htobe64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) // CROARING_IS_BIG_ENDIAN +#include +#define croaring_htobe64(x) OSSwapInt64(x) + +#elif defined(__has_include) && \ + __has_include( \ + ) && (defined(__linux__) || defined(__FreeBSD__)) // CROARING_IS_BIG_ENDIAN +#include +#if defined(__linux__) +#define croaring_htobe64(x) bswap_64(x) +#elif defined(__FreeBSD__) +#define croaring_htobe64(x) bswap64(x) +#else +#warning "Unknown platform, report as an error" +#endif + +#else // CROARING_IS_BIG_ENDIAN +// Gets compiled to bswap or equivalent on most compilers. +#define croaring_htobe64(x) \ + (((x & 0x00000000000000FFULL) << 56) | \ + ((x & 0x000000000000FF00ULL) << 40) | \ + ((x & 0x0000000000FF0000ULL) << 24) | \ + ((x & 0x00000000FF000000ULL) << 8) | ((x & 0x000000FF00000000ULL) >> 8) | \ + ((x & 0x0000FF0000000000ULL) >> 24) | \ + ((x & 0x00FF000000000000ULL) >> 40) | \ + ((x & 0xFF00000000000000ULL) >> 56)) +#endif // CROARING_IS_BIG_ENDIAN +#define croaring_be64toh(x) croaring_htobe64(x) +// End of host <-> big endian conversion. + // Defines for the possible CROARING atomic implementations -#define CROARING_ATOMIC_IMPL_NONE 1 -#define CROARING_ATOMIC_IMPL_CPP 2 -#define CROARING_ATOMIC_IMPL_C 3 -#define CROARING_ATOMIC_IMPL_C_WINDOWS 4 +#define CROARING_ATOMIC_IMPL_NONE 1 +#define CROARING_ATOMIC_IMPL_CPP 2 +#define CROARING_ATOMIC_IMPL_C 3 +#define CROARING_ATOMIC_IMPL_C_WINDOWS 4 // If the use has forced a specific implementation, use that, otherwise, // figure out the best implementation we can use. #if !defined(CROARING_ATOMIC_IMPL) - #if defined(__cplusplus) && __cplusplus >= 201103L - #ifdef __has_include - #if __has_include() - #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP - #endif //__has_include() - #else - // We lack __has_include to check: - #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP - #endif //__has_include - #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) - #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C - #elif CROARING_REGULAR_VISUAL_STUDIO - // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ - #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS - #endif -#endif // !defined(CROARING_ATOMIC_IMPL) +#if defined(__cplusplus) && __cplusplus >= 201103L +#ifdef __has_include +#if __has_include() +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP +#endif //__has_include() +#else + // We lack __has_include to check: +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP +#endif //__has_include +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C +#elif CROARING_REGULAR_VISUAL_STUDIO + // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS +#endif +#endif // !defined(CROARING_ATOMIC_IMPL) #if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C #include @@ -613,17 +682,18 @@ typedef _Atomic(uint32_t) croaring_refcount_t; static inline void croaring_refcount_inc(croaring_refcount_t *val) { // Increasing the reference counter can always be done with // memory_order_relaxed: New references to an object can only be formed from - // an existing reference, and passing an existing reference from one thread to - // another must already provide any required synchronization. + // an existing reference, and passing an existing reference from one thread + // to another must already provide any required synchronization. atomic_fetch_add_explicit(val, 1, memory_order_relaxed); } static inline bool croaring_refcount_dec(croaring_refcount_t *val) { - // It is important to enforce any possible access to the object in one thread - // (through an existing reference) to happen before deleting the object in a - // different thread. This is achieved by a "release" operation after dropping - // a reference (any access to the object through this reference must obviously - // happened before), and an "acquire" operation before deleting the object. + // It is important to enforce any possible access to the object in one + // thread (through an existing reference) to happen before deleting the + // object in a different thread. This is achieved by a "release" operation + // after dropping a reference (any access to the object through this + // reference must obviously happened before), and an "acquire" operation + // before deleting the object. bool is_zero = atomic_fetch_sub_explicit(val, 1, memory_order_release) == 1; if (is_zero) { atomic_thread_fence(memory_order_acquire); @@ -672,7 +742,8 @@ static inline bool croaring_refcount_dec(croaring_refcount_t *val) { } static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { - // Per https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access + // Per + // https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access // > Simple reads and writes to properly-aligned 32-bit variables are atomic // > operations. In other words, you will not end up with only one portion // > of the variable updated; all bits are updated in an atomic fashion. @@ -699,6 +770,11 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #error "Unknown atomic implementation" #endif +#if defined(__GNUC__) || defined(__clang__) +#define CROARING_DEPRECATED __attribute__((deprecated)) +#else +#define CROARING_DEPRECATED +#endif // defined(__GNUC__) || defined(__clang__) // We need portability.h to be included first, // but we also always want isadetection.h to be @@ -707,18 +783,18 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { // There is no scenario where we want portability.h to // be included, but not isadetection.h: the latter is a // strict requirement. -#endif /* INCLUDE_PORTABILITY_H_ */ +#endif /* INCLUDE_PORTABILITY_H_ */ /* end file include/roaring/portability.h */ /* begin file include/roaring/bitset/bitset.h */ -#ifndef CBITSET_BITSET_H -#define CBITSET_BITSET_H +#ifndef CROARING_CBITSET_BITSET_H +#define CROARING_CBITSET_BITSET_H // For compatibility with MSVC with the use of `restrict` #if (__STDC_VERSION__ >= 199901L) || \ (defined(__GNUC__) && defined(__STDC_VERSION__)) -#define CBITSET_RESTRICT restrict +#define CROARING_CBITSET_RESTRICT restrict #else -#define CBITSET_RESTRICT +#define CROARING_CBITSET_RESTRICT #endif // (__STDC_VERSION__ >= 199901L) || (defined(__GNUC__) && // defined(__STDC_VERSION__ )) @@ -728,14 +804,18 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #include #include + #ifdef __cplusplus -extern "C" { namespace roaring { namespace api { +extern "C" { +namespace roaring { +namespace api { #endif struct bitset_s { - uint64_t *CBITSET_RESTRICT array; - /* For simplicity and performance, we prefer to have a size and a capacity that is a multiple of 64 bits. - * Thus we only track the size and the capacity in terms of 64-bit words allocated */ + uint64_t *CROARING_CBITSET_RESTRICT array; + /* For simplicity and performance, we prefer to have a size and a capacity + * that is a multiple of 64 bits. Thus we only track the size and the + * capacity in terms of 64-bit words allocated */ size_t arraysize; size_t capacity; }; @@ -761,9 +841,9 @@ void bitset_fill(bitset_t *bitset); /* Create a copy */ bitset_t *bitset_copy(const bitset_t *bitset); -/* For advanced users: Resize the bitset so that it can support newarraysize * 64 bits. - * Return true in case of success, false for failure. Pad - * with zeroes new buffer areas if requested. */ +/* For advanced users: Resize the bitset so that it can support newarraysize * + * 64 bits. Return true in case of success, false for failure. Pad with zeroes + * new buffer areas if requested. */ bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes); /* returns how many bytes of memory the backend buffer uses */ @@ -781,12 +861,12 @@ inline size_t bitset_size_in_words(const bitset_t *bitset) { return bitset->arraysize; } -/* For advanced users: Grow the bitset so that it can support newarraysize * 64 bits with padding. - * Return true in case of success, false for failure. */ +/* For advanced users: Grow the bitset so that it can support newarraysize * 64 + * bits with padding. Return true in case of success, false for failure. */ bool bitset_grow(bitset_t *bitset, size_t newarraysize); -/* attempts to recover unused memory, return false in case of roaring_reallocation - * failure */ +/* attempts to recover unused memory, return false in case of + * roaring_reallocation failure */ bool bitset_trim(bitset_t *bitset); /* shifts all bits by 's' positions so that the bitset representing values @@ -846,48 +926,53 @@ size_t bitset_maximum(const bitset_t *bitset); /* compute the union in-place (to b1), returns true if successful, to generate a * new bitset first call bitset_copy */ -bool bitset_inplace_union(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +bool bitset_inplace_union(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* report the size of the union (without materializing it) */ -size_t bitset_union_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +size_t bitset_union_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* compute the intersection in-place (to b1), to generate a new bitset first * call bitset_copy */ -void bitset_inplace_intersection(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +void bitset_inplace_intersection(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* report the size of the intersection (without materializing it) */ -size_t bitset_intersection_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +size_t bitset_intersection_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* returns true if the bitsets contain no common elements */ -bool bitsets_disjoint(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2); +bool bitsets_disjoint(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* returns true if the bitsets contain any common elements */ -bool bitsets_intersect(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2); +bool bitsets_intersect(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* returns true if b1 contains all of the set bits of b2 */ -bool bitset_contains_all(const bitset_t *CBITSET_RESTRICT b1, const bitset_t *CBITSET_RESTRICT b2); +bool bitset_contains_all(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* compute the difference in-place (to b1), to generate a new bitset first call * bitset_copy */ -void bitset_inplace_difference(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +void bitset_inplace_difference(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* compute the size of the difference */ -size_t bitset_difference_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +size_t bitset_difference_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* compute the symmetric difference in-place (to b1), return true if successful, * to generate a new bitset first call bitset_copy */ -bool bitset_inplace_symmetric_difference(bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +bool bitset_inplace_symmetric_difference( + bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* compute the size of the symmetric difference */ -size_t bitset_symmetric_difference_count(const bitset_t *CBITSET_RESTRICT b1, - const bitset_t *CBITSET_RESTRICT b2); +size_t bitset_symmetric_difference_count( + const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2); /* iterate over the set bits like so : @@ -922,8 +1007,8 @@ inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i) { like so : size_t buffer[256]; size_t howmany = 0; - for(size_t startfrom = 0; (howmany = bitset_next_set_bits(b,buffer,256, &startfrom)) > - 0 ; startfrom++) { + for(size_t startfrom = 0; (howmany = bitset_next_set_bits(b,buffer,256, + &startfrom)) > 0 ; startfrom++) { //..... } */ @@ -988,7 +1073,9 @@ inline void bitset_print(const bitset_t *b) { } #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace api { +} +} +} // extern "C" { namespace roaring { namespace api { #endif #endif @@ -1002,12 +1089,16 @@ inline void bitset_print(const bitset_t *b) { #define ROARING_H #include -#include #include // for `size_t` +#include + +// Include other headers after roaring_types.h #ifdef __cplusplus -extern "C" { namespace roaring { namespace api { +extern "C" { +namespace roaring { +namespace api { #endif typedef struct roaring_bitmap_s { @@ -1027,8 +1118,9 @@ roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap); * Returns NULL if the allocation fails. * Client is responsible for calling `roaring_bitmap_free()`. */ -inline roaring_bitmap_t *roaring_bitmap_create(void) - { return roaring_bitmap_create_with_capacity(0); } +inline roaring_bitmap_t *roaring_bitmap_create(void) { + return roaring_bitmap_create_with_capacity(0); +} /** * Initialize a roaring bitmap structure in memory controlled by client. @@ -1042,13 +1134,14 @@ bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap); * The bitmap will be in a "clear" state, with no auxiliary allocations. * Since this performs no allocations, the function will not fail. */ -inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r) - { roaring_bitmap_init_with_capacity(r, 0); } +inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r) { + roaring_bitmap_init_with_capacity(r, 0); +} /** * Add all the values between min (included) and max (excluded) that are at a * distance k*step from min. -*/ + */ roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, uint32_t step); @@ -1066,10 +1159,10 @@ roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals); * do so for all of your bitmaps, since interactions between bitmaps with and * without COW is unsafe. */ -inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t* r) { +inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r) { return r->high_low_container.flags & ROARING_FLAG_COW; } -inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t* r, bool cow) { +inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, bool cow) { if (cow) { r->high_low_container.flags |= ROARING_FLAG_COW; } else { @@ -1086,8 +1179,51 @@ void roaring_bitmap_printf_describe(const roaring_bitmap_t *r); /** * Creates a new bitmap from a list of uint32_t integers + * + * This function is deprecated, use `roaring_bitmap_from` instead, which + * doesn't require the number of elements to be passed in. + * + * @see roaring_bitmap_from */ -roaring_bitmap_t *roaring_bitmap_of(size_t n, ...); +CROARING_DEPRECATED roaring_bitmap_t *roaring_bitmap_of(size_t n, ...); + +#ifdef __cplusplus +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring_bitmap_of_ptr` function instead. + */ +// Use an immediately invoked closure, capturing by reference +// (in case __VA_ARGS__ refers to context outside the closure) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring_bitmap_from(...) \ + [&]() { \ + const uint32_t roaring_bitmap_from_array[] = {0, __VA_ARGS__}; \ + return roaring_bitmap_of_ptr((sizeof(roaring_bitmap_from_array) / \ + sizeof(roaring_bitmap_from_array[0])) - \ + 1, \ + &roaring_bitmap_from_array[1]); \ + }() +#else +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring_bitmap_of_ptr` function instead. + */ +// While __VA_ARGS__ occurs twice in expansion, one of the times is in a sizeof +// expression, which is an unevaluated context, so it's even safe in the case +// where expressions passed have side effects (roaring64_bitmap_from(my_func(), +// ++i)) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring_bitmap_from(...) \ + roaring_bitmap_of_ptr( \ + (sizeof((const uint32_t[]){0, __VA_ARGS__}) / sizeof(uint32_t)) - 1, \ + &((const uint32_t[]){0, __VA_ARGS__})[1]) +#endif /** * Copies a bitmap (this does memory allocation). @@ -1104,7 +1240,8 @@ roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r); * that roaring_bitmap_overwrite can save on memory allocations. * * Returns true if successful, or false if there was an error. On failure, - * the dest bitmap is left in a valid, empty state (even if it was not empty before). + * the dest bitmap is left in a valid, empty state (even if it was not empty + * before). */ bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, const roaring_bitmap_t *src); @@ -1139,10 +1276,10 @@ bool roaring_bitmap_intersect(const roaring_bitmap_t *r1, const roaring_bitmap_t *r2); /** - * Check whether a bitmap and a closed range intersect. + * Check whether a bitmap and an open range intersect. */ -bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, - uint64_t x, uint64_t y); +bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, + uint64_t y); /** * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto @@ -1320,15 +1457,15 @@ bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t x); /** * Add all values in range [min, max] */ -void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, - uint32_t min, uint32_t max); +void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max); /** * Add all values in range [min, max) */ -inline void roaring_bitmap_add_range(roaring_bitmap_t *r, - uint64_t min, uint64_t max) { - if(max <= min) return; +inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max) { + if (max <= min) return; roaring_bitmap_add_range_closed(r, (uint32_t)min, (uint32_t)(max - 1)); } @@ -1340,15 +1477,15 @@ void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t x); /** * Remove all values in range [min, max] */ -void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, - uint32_t min, uint32_t max); +void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max); /** * Remove all values in range [min, max) */ -inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, - uint64_t min, uint64_t max) { - if(max <= min) return; +inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max) { + if (max <= min) return; roaring_bitmap_remove_range_closed(r, (uint32_t)min, (uint32_t)(max - 1)); } @@ -1374,8 +1511,7 @@ bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val); * to range_end (excluded) is present */ bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, - uint64_t range_start, - uint64_t range_end); + uint64_t range_start, uint64_t range_end); /** * Check if an items is present, using context from a previous insert or search @@ -1409,11 +1545,10 @@ uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, uint64_t range_end); /** -* Returns true if the bitmap is empty (cardinality is zero). -*/ + * Returns true if the bitmap is empty (cardinality is zero). + */ bool roaring_bitmap_is_empty(const roaring_bitmap_t *r); - /** * Empties the bitmap. It will have no auxiliary allocations (so if the bitmap * was initialized in client memory via roaring_bitmap_init(), then a call to @@ -1438,7 +1573,7 @@ void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans); * * bitset_t * out = bitset_create(); * // if the bitset has content in it, call "bitset_clear(out)" - * bool success = roaring_bitmap_to_bitset(mybitmap, out); + * bool success = roaring_bitmap_to_bitset(mybitmap, out); * // on failure, success will be false. * // You can then query the bitset: * bool is_present = bitset_get(out, 10011 ); @@ -1446,10 +1581,11 @@ void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans); * bitset_free(out); * */ -bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t * bitset); +bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset); /** - * Convert the bitmap to a sorted array from `offset` by `limit`, output in `ans`. + * Convert the bitmap to a sorted array from `offset` by `limit`, output in + * `ans`. * * Caller is responsible to ensure that there is enough memory allocated, e.g. * @@ -1457,9 +1593,8 @@ bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t * bitset); * * Return false in case of failure (e.g., insufficient memory) */ -bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, - size_t offset, size_t limit, - uint32_t *ans); +bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, + size_t limit, uint32_t *ans); /** * Remove run-length encoding even when it is more space efficient. @@ -1492,8 +1627,9 @@ size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r); * * Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); @@ -1503,8 +1639,9 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); * (See `roaring_bitmap_portable_deserialize()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); @@ -1514,13 +1651,16 @@ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); * (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. - * - * The difference with `roaring_bitmap_deserialize()` is that this function checks that the input buffer - * is a valid bitmap. If the buffer is too small, NULL is returned. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + * + * The difference with `roaring_bitmap_deserialize()` is that this function + * checks that the input buffer is a valid bitmap. If the buffer is too small, + * NULL is returned. */ -roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, size_t maxbytes); +roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, + size_t maxbytes); /** * How many bytes are required to serialize this bitmap (NOT compatible @@ -1538,9 +1678,10 @@ size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r); * * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec -* - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); @@ -1551,17 +1692,23 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * The function itself is safe in the sense that it will not cause buffer overflows. - * However, for correct operations, it is assumed that the bitmap read was once - * serialized from a valid bitmap (i.e., it follows the format specification). - * If you provided an incorrect input (garbage), then the bitmap read may not be in - * a valid state and following operations may not lead to sensible results. - * In particular, the serialized array containers need to be in sorted order, and the - * run containers should be in sorted non-overlapping order. This is is guaranteed to - * happen when serializing an existing bitmap, but not for random inputs. + * The function itself is safe in the sense that it will not cause buffer + * overflows. However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results. In particular, the serialized array containers need to be + * in sorted order, and the run containers should be in sorted non-overlapping + * order. This is is guaranteed to happen when serializing an existing bitmap, + * but not for random inputs. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * You may use roaring_bitmap_internal_validate to check the validity of the + * bitmap prior to using it. You may also use other strategies to check for + * corrupted inputs (e.g., checksums). + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, size_t maxbytes); @@ -1582,8 +1729,9 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf); @@ -1615,8 +1763,9 @@ size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r); * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf); @@ -1648,8 +1797,9 @@ size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *r); * Serializes bitmap using frozen format. * Buffer size must be at least roaring_bitmap_frozen_size_in_bytes(). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); @@ -1664,8 +1814,9 @@ void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); * Bitmap must be freed as usual, by calling roaring_bitmap_free(). * Underlying buffer must not be freed or modified while it backs any bitmaps. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a mainframe IBM s390x), - * the data format is going to be big-endian and not compatible with little-endian systems. + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. */ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, size_t length); @@ -1807,6 +1958,18 @@ bool roaring_bitmap_select(const roaring_bitmap_t *r, uint32_t rank, */ uint64_t roaring_bitmap_rank(const roaring_bitmap_t *r, uint32_t x); +/** + * roaring_bitmap_rank_many is an `Bulk` version of `roaring_bitmap_rank` + * it puts rank value of each element in `[begin .. end)` to `ans[]` + * + * the values in `[begin .. end)` must be sorted in Ascending order; + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * + * ans = malloc((end-begin) * sizeof(uint64_t)); + */ +void roaring_bitmap_rank_many(const roaring_bitmap_t *r, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + /** * Returns the index of x in the given roaring bitmap. * If the roaring bitmap doesn't contain x , this function will return -1. @@ -1836,145 +1999,214 @@ void roaring_bitmap_statistics(const roaring_bitmap_t *r, roaring_statistics_t *stat); /** - * Perform internal consistency checks. Returns true if the bitmap is consistent. + * Perform internal consistency checks. Returns true if the bitmap is + * consistent. It may be useful to call this after deserializing bitmaps from + * untrusted sources. If roaring_bitmap_internal_validate returns true, then the + * bitmap should be consistent and can be trusted not to cause crashes or memory + * corruption. * - * Note that some operations intentionally leave bitmaps in an inconsistent state temporarily, - * for example, `roaring_bitmap_lazy_*` functions, until `roaring_bitmap_repair_after_lazy` is called. + * Note that some operations intentionally leave bitmaps in an inconsistent + * state temporarily, for example, `roaring_bitmap_lazy_*` functions, until + * `roaring_bitmap_repair_after_lazy` is called. * - * If reason is non-null, it will be set to a string describing the first inconsistency found if any. + * If reason is non-null, it will be set to a string describing the first + * inconsistency found if any. */ -bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, const char **reason); +bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, + const char **reason); /********************* * What follows is code use to iterate through values in a roaring bitmap roaring_bitmap_t *r =... roaring_uint32_iterator_t i; -roaring_create_iterator(r, &i); +roaring_iterator_create(r, &i); while(i.has_value) { printf("value = %d\n", i.current_value); - roaring_advance_uint32_iterator(&i); + roaring_uint32_iterator_advance(&i); } Obviously, if you modify the underlying bitmap, the iterator becomes invalid. So don't. */ +/** + * A struct used to keep iterator state. Users should only access + * `current_value` and `has_value`, the rest of the type should be treated as + * opaque. + */ typedef struct roaring_uint32_iterator_s { - const roaring_bitmap_t *parent; // owner - int32_t container_index; // point to the current container index - int32_t in_container_index; // for bitset and array container, this is out - // index - int32_t run_index; // for run container, this points at the run + const roaring_bitmap_t *parent; // Owner + const ROARING_CONTAINER_T *container; // Current container + uint8_t typecode; // Typecode of current container + int32_t container_index; // Current container index + uint32_t highbits; // High 16 bits of the current value + roaring_container_iterator_t container_it; uint32_t current_value; bool has_value; - - const ROARING_CONTAINER_T - *container; // should be: - // parent->high_low_container.containers[container_index]; - uint8_t typecode; // should be: - // parent->high_low_container.typecodes[container_index]; - uint32_t highbits; // should be: - // parent->high_low_container.keys[container_index]) << - // 16; - } roaring_uint32_iterator_t; /** - * Initialize an iterator object that can be used to iterate through the - * values. If there is a value, then this iterator points to the first value - * and `it->has_value` is true. The value is in `it->current_value`. + * Initialize an iterator object that can be used to iterate through the values. + * If there is a value, then this iterator points to the first value and + * `it->has_value` is true. The value is in `it->current_value`. */ -void roaring_init_iterator(const roaring_bitmap_t *r, +void roaring_iterator_init(const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit); +/** DEPRECATED, use `roaring_iterator_init`. */ +CROARING_DEPRECATED static inline void roaring_init_iterator( + const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit) { + roaring_iterator_init(r, newit); +} + /** - * Initialize an iterator object that can be used to iterate through the - * values. If there is a value, then this iterator points to the last value - * and `it->has_value` is true. The value is in `it->current_value`. + * Initialize an iterator object that can be used to iterate through the values. + * If there is a value, then this iterator points to the last value and + * `it->has_value` is true. The value is in `it->current_value`. */ -void roaring_init_iterator_last(const roaring_bitmap_t *r, +void roaring_iterator_init_last(const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit); +/** DEPRECATED, use `roaring_iterator_init_last`. */ +CROARING_DEPRECATED static inline void roaring_init_iterator_last( + const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit) { + roaring_iterator_init_last(r, newit); +} + /** * Create an iterator object that can be used to iterate through the values. * Caller is responsible for calling `roaring_free_iterator()`. * - * The iterator is initialized (this function calls `roaring_init_iterator()`) + * The iterator is initialized (this function calls `roaring_iterator_init()`) * If there is a value, then this iterator points to the first value and * `it->has_value` is true. The value is in `it->current_value`. */ -roaring_uint32_iterator_t *roaring_create_iterator(const roaring_bitmap_t *r); +roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r); + +/** DEPRECATED, use `roaring_iterator_create`. */ +CROARING_DEPRECATED static inline roaring_uint32_iterator_t * +roaring_create_iterator(const roaring_bitmap_t *r) { + return roaring_iterator_create(r); +} /** -* Advance the iterator. If there is a new value, then `it->has_value` is true. -* The new value is in `it->current_value`. Values are traversed in increasing -* orders. For convenience, returns `it->has_value`. -*/ -bool roaring_advance_uint32_iterator(roaring_uint32_iterator_t *it); + * Advance the iterator. If there is a new value, then `it->has_value` is true. + * The new value is in `it->current_value`. Values are traversed in increasing + * orders. For convenience, returns `it->has_value`. + * + * Once `it->has_value` is false, `roaring_uint32_iterator_advance` should not + * be called on the iterator again. Calling `roaring_uint32_iterator_previous` + * is allowed. + */ +bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_advance`. */ +CROARING_DEPRECATED static inline bool roaring_advance_uint32_iterator( + roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_advance(it); +} /** -* Decrement the iterator. If there's a new value, then `it->has_value` is true. -* The new value is in `it->current_value`. Values are traversed in decreasing -* order. For convenience, returns `it->has_value`. -*/ -bool roaring_previous_uint32_iterator(roaring_uint32_iterator_t *it); + * Decrement the iterator. If there's a new value, then `it->has_value` is true. + * The new value is in `it->current_value`. Values are traversed in decreasing + * order. For convenience, returns `it->has_value`. + * + * Once `it->has_value` is false, `roaring_uint32_iterator_previous` should not + * be called on the iterator again. Calling `roaring_uint32_iterator_advance` is + * allowed. + */ +bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_previous`. */ +CROARING_DEPRECATED static inline bool roaring_previous_uint32_iterator( + roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_previous(it); +} /** * Move the iterator to the first value >= `val`. If there is a such a value, * then `it->has_value` is true. The new value is in `it->current_value`. * For convenience, returns `it->has_value`. */ -bool roaring_move_uint32_iterator_equalorlarger(roaring_uint32_iterator_t *it, +bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, uint32_t val); +/** DEPRECATED, use `roaring_uint32_iterator_move_equalorlarger`. */ +CROARING_DEPRECATED static inline bool +roaring_move_uint32_iterator_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val) { + return roaring_uint32_iterator_move_equalorlarger(it, val); +} + /** * Creates a copy of an iterator. * Caller must free it. */ -roaring_uint32_iterator_t *roaring_copy_uint32_iterator( +roaring_uint32_iterator_t *roaring_uint32_iterator_copy( const roaring_uint32_iterator_t *it); +/** DEPRECATED, use `roaring_uint32_iterator_copy`. */ +CROARING_DEPRECATED static inline roaring_uint32_iterator_t * +roaring_copy_uint32_iterator(const roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_copy(it); +} + /** - * Free memory following `roaring_create_iterator()` + * Free memory following `roaring_iterator_create()` */ -void roaring_free_uint32_iterator(roaring_uint32_iterator_t *it); +void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_free`. */ +CROARING_DEPRECATED static inline void roaring_free_uint32_iterator( + roaring_uint32_iterator_t *it) { + roaring_uint32_iterator_free(it); +} /* * Reads next ${count} values from iterator into user-supplied ${buf}. * Returns the number of read elements. - * This number can be smaller than ${count}, which means that iterator is drained. + * This number can be smaller than ${count}, which means that iterator is + * drained. * * This function satisfies semantics of iteration and can be used together with * other iterator functions. * - first value is copied from ${it}->current_value * - after function returns, iterator is positioned at the next element */ -uint32_t roaring_read_uint32_iterator(roaring_uint32_iterator_t *it, - uint32_t* buf, uint32_t count); +uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count); + +/** DEPRECATED, use `roaring_uint32_iterator_read`. */ +CROARING_DEPRECATED static inline uint32_t roaring_read_uint32_iterator( + roaring_uint32_iterator_t *it, uint32_t *buf, uint32_t count) { + return roaring_uint32_iterator_read(it, buf, count); +} #ifdef __cplusplus -} } } // extern "C" { namespace roaring { namespace api { +} +} +} // extern "C" { namespace roaring { namespace api { #endif -#endif /* ROARING_H */ +#endif /* ROARING_H */ #ifdef __cplusplus - /** - * Best practices for C++ headers is to avoid polluting global scope. - * But for C compatibility when just `roaring.h` is included building as - * C++, default to global access for the C public API. - * - * BUT when `roaring.hh` is included instead, it sets this flag. That way - * explicit namespacing must be used to get the C functions. - * - * This is outside the include guard so that if you include BOTH headers, - * the order won't matter; you still get the global definitions. - */ - #if !defined(ROARING_API_NOT_IN_GLOBAL_NAMESPACE) - using namespace ::roaring::api; - #endif +/** + * Best practices for C++ headers is to avoid polluting global scope. + * But for C compatibility when just `roaring.h` is included building as + * C++, default to global access for the C public API. + * + * BUT when `roaring.hh` is included instead, it sets this flag. That way + * explicit namespacing must be used to get the C functions. + * + * This is outside the include guard so that if you include BOTH headers, + * the order won't matter; you still get the global definitions. + */ +#if !defined(ROARING_API_NOT_IN_GLOBAL_NAMESPACE) +using namespace ::roaring::api; +#endif #endif /* end file include/roaring/roaring.h */ /* begin file include/roaring/memory.h */ @@ -2018,3 +2250,659 @@ void roaring_aligned_free(void*); #endif // INCLUDE_ROARING_MEMORY_H_ /* end file include/roaring/memory.h */ +/* begin file include/roaring/roaring64.h */ +#ifndef ROARING64_H +#define ROARING64_H + +#include +#include +#include + + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace api { +#endif + +typedef struct roaring64_bitmap_s roaring64_bitmap_t; +typedef struct roaring64_leaf_s roaring64_leaf_t; +typedef struct roaring64_iterator_s roaring64_iterator_t; + +/** + * A bit of context usable with `roaring64_bitmap_*_bulk()` functions. + * + * Should be initialized with `{0}` (or `memset()` to all zeros). + * Callers should treat it as an opaque type. + * + * A context may only be used with a single bitmap (unless re-initialized to + * zero), and any modification to a bitmap (other than modifications performed + * with `_bulk()` functions with the context passed) will invalidate any + * contexts associated with that bitmap. + */ +typedef struct roaring64_bulk_context_s { + uint8_t high_bytes[6]; + roaring64_leaf_t *leaf; +} roaring64_bulk_context_t; + +/** + * Dynamically allocates a new bitmap (initially empty). + * Client is responsible for calling `roaring64_bitmap_free()`. + */ +roaring64_bitmap_t *roaring64_bitmap_create(void); +void roaring64_bitmap_free(roaring64_bitmap_t *r); + +/** + * Returns a copy of a bitmap. + */ +roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r); + +/** + * Creates a new bitmap of a pointer to N 64-bit integers. + */ +roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, + const uint64_t *vals); + +#ifdef __cplusplus +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring64_bitmap_of_ptr` function instead. + */ +// Use an immediately invoked closure, capturing by reference +// (in case __VA_ARGS__ refers to context outside the closure) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring64_bitmap_from(...) \ + [&]() { \ + const uint64_t roaring64_bitmap_from_array[] = {0, __VA_ARGS__}; \ + return roaring64_bitmap_of_ptr( \ + (sizeof(roaring64_bitmap_from_array) / \ + sizeof(roaring64_bitmap_from_array[0])) - \ + 1, \ + &roaring64_bitmap_from_array[1]); \ + }() +#else +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring64_bitmap_of_ptr` function instead. + */ +// While __VA_ARGS__ occurs twice in expansion, one of the times is in a sizeof +// expression, which is an unevaluated context, so it's even safe in the case +// where expressions passed have side effects (roaring64_bitmap_from(my_func(), +// ++i)) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring64_bitmap_from(...) \ + roaring64_bitmap_of_ptr( \ + (sizeof((const uint64_t[]){0, __VA_ARGS__}) / sizeof(uint64_t)) - 1, \ + &((const uint64_t[]){0, __VA_ARGS__})[1]) +#endif + +/** + * Create a new bitmap containing all the values in [min, max) that are at a + * distance k*step from min. + */ +roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, + uint64_t step); + +/** + * Adds the provided value to the bitmap. + */ +void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val); + +/** + * Adds the provided value to the bitmap. + * Returns true if a new value was added, false if the value already existed. + */ +bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val); + +/** + * Add an item, using context from a previous insert for faster insertion. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, uint64_t val); + +/** + * Add `n_args` values from `vals`, faster than repeatedly calling + * `roaring64_bitmap_add()` + * + * In order to exploit this optimization, the caller should attempt to keep + * values with the same high 48 bits of the value as consecutive elements in + * `vals`. + */ +void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals); + +/** + * Add all values in range [min, max). + */ +void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Add all values in range [min, max]. + */ +void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Removes a value from the bitmap if present. + */ +void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val); + +/** + * Removes a value from the bitmap if present, returns true if the value was + * removed and false if the value was not present. + */ +bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val); + +/** + * Remove an item, using context from a previous insert for faster removal. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val); + +/** + * Remove `n_args` values from `vals`, faster than repeatedly calling + * `roaring64_bitmap_remove()` + * + * In order to exploit this optimization, the caller should attempt to keep + * values with the same high 48 bits of the value as consecutive elements in + * `vals`. + */ +void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals); + +/** + * Remove all values in range [min, max). + */ +void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Remove all values in range [min, max]. + */ +void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Returns true if the provided value is present. + */ +bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val); + +/** + * Returns true if all values in the range [min, max) are present. + */ +bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Check if an item is present using context from a previous insert or search + * for faster search. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val); + +/** + * Selects the element at index 'rank' where the smallest element is at index 0. + * If the size of the bitmap is strictly greater than rank, then this function + * returns true and sets element to the element of given rank. Otherwise, it + * returns false. + */ +bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, + uint64_t *element); + +/** + * Returns the number of integers that are smaller or equal to x. Thus if x is + * the first element, this function will return 1. If x is smaller than the + * smallest element, this function will return 0. + * + * The indexing convention differs between roaring64_bitmap_select and + * roaring64_bitmap_rank: roaring_bitmap64_select refers to the smallest value + * as having index 0, whereas roaring64_bitmap_rank returns 1 when ranking + * the smallest value. + */ +uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val); + +/** + * Returns true if the given value is in the bitmap, and sets `out_index` to the + * (0-based) index of the value in the bitmap. Returns false if the value is not + * in the bitmap. + */ +bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, + uint64_t *out_index); + +/** + * Returns the number of values in the bitmap. + */ +uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r); + +/** + * Returns the number of elements in the range [min, max). + */ +uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Returns true if the bitmap is empty (cardinality is zero). + */ +bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r); + +/** + * Returns the smallest value in the set, or UINT64_MAX if the set is empty. + */ +uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r); + +/** + * Returns the largest value in the set, or 0 if empty. + */ +uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r); + +/** + * Returns true if the result has at least one run container. + */ +bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r); + +/** + * Perform internal consistency checks. + * + * Returns true if the bitmap is consistent. It may be useful to call this + * after deserializing bitmaps from untrusted sources. If + * roaring64_bitmap_internal_validate returns true, then the bitmap is + * consistent and can be trusted not to cause crashes or memory corruption. + * + * If reason is non-null, it will be set to a string describing the first + * inconsistency found if any. + */ +bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, + const char **reason); + +/** + * Return true if the two bitmaps contain the same elements. + */ +bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2. + */ +bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2, and r2 is strictly + * greater than r1. + */ +bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the intersection between two bitmaps and returns new bitmap. The + * caller is responsible for free-ing the result. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmaps. You may + * also rely on roaring64_bitmap_and_inplace to avoid creating many temporary + * bitmaps. + */ +roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the intersection between two bitmaps. + */ +uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_and()`, modifies `r1`. `r1` and `r2` + * are allowed to be equal. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmaps. + */ +void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Check whether two bitmaps intersect. + */ +bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Check whether a bitmap intersects the range [min, max). + */ +bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto + * distance, or the Jaccard similarity coefficient) + * + * The Jaccard index is undefined if both bitmaps are empty. + */ +double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the union between two bitmaps and returns new bitmap. The caller is + * responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the union between two bitmaps. + */ +uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_or(), modifies `r1`. + */ +void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the symmetric difference (xor) between two bitmaps and returns a new + * bitmap. The caller is responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the symmetric difference (xor) between two bitmaps. + */ +uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_xor()`, modifies `r1`. `r1` and `r2` + * are not allowed to be equal (that would result in an empty bitmap). + */ +void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the difference (andnot) between two bitmaps and returns a new + * bitmap. The caller is responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the difference (andnot) between two bitmaps. + */ +uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_andnot()`, modifies `r1`. `r1` and `r2` + * are not allowed to be equal (that would result in an empty bitmap). + */ +void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Compute the negation of the bitmap in the interval [min, max). + * The number of negated values is `max - min`. Areas outside the range are + * passed through unchanged. + */ +roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Compute the negation of the bitmap in the interval [min, max]. + * The number of negated values is `max - min + 1`. Areas outside the range are + * passed through unchanged. + */ +roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * In-place version of `roaring64_bitmap_flip`. Compute the negation of the + * bitmap in the interval [min, max). The number of negated values is `max - + * min`. Areas outside the range are passed through unchanged. + */ +void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); +/** + * In-place version of `roaring64_bitmap_flip_closed`. Compute the negation of + * the bitmap in the interval [min, max]. The number of negated values is `max - + * min + 1`. Areas outside the range are passed through unchanged. + */ +void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); +/** + * How many bytes are required to serialize this bitmap. + * + * This is meant to be compatible with other languages: + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + */ +size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r); + +/** + * Write a bitmap to a buffer. The output buffer should refer to at least + * `roaring64_bitmap_portable_size_in_bytes(r)` bytes of allocated memory. + * + * Returns how many bytes were written, which should match + * `roaring64_bitmap_portable_size_in_bytes(r)`. + * + * This is meant to be compatible with other languages: + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, + char *buf); +/** + * Check how many bytes would be read (up to maxbytes) at this pointer if there + * is a valid bitmap, returns zero if there is no valid bitmap. + * + * This is meant to be compatible with other languages + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + */ +size_t roaring64_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes); + +/** + * Read a bitmap from a serialized buffer safely (reading up to maxbytes). + * In case of failure, NULL is returned. + * + * This is meant to be compatible with other languages + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + * + * The function itself is safe in the sense that it will not cause buffer + * overflows. However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results. In particular, the serialized array containers need to be + * in sorted order, and the run containers should be in sorted non-overlapping + * order. This is is guaranteed to happen when serializing an existing bitmap, + * but not for random inputs. + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes); + +/** + * Iterate over the bitmap elements. The function `iterator` is called once for + * all the values with `ptr` (can be NULL) as the second parameter of each call. + * + * `roaring_iterator64` is simply a pointer to a function that returns a bool + * and takes `(uint64_t, void*)` as inputs. True means that the iteration should + * continue, while false means that it should stop. + * + * Returns true if the `roaring64_iterator` returned true throughout (so that + * all data points were necessarily visited). + * + * Iteration is ordered from the smallest to the largest elements. + */ +bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, + roaring_iterator64 iterator, void *ptr); + +/** + * Convert the bitmap to a sorted array `out`. + * + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * ``` + * out = malloc(roaring64_bitmap_get_cardinality(bitmap) * sizeof(uint64_t)); + * ``` + */ +void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, + uint64_t *out); + +/** + * Create an iterator object that can be used to iterate through the values. + * Caller is responsible for calling `roaring64_iterator_free()`. + * + * The iterator is initialized. If there is a value, then this iterator points + * to the first value and `roaring64_iterator_has_value()` returns true. The + * value can be retrieved with `roaring64_iterator_value()`. + */ +roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r); + +/** + * Create an iterator object that can be used to iterate through the values. + * Caller is responsible for calling `roaring64_iterator_free()`. + * + * The iterator is initialized. If there is a value, then this iterator points + * to the last value and `roaring64_iterator_has_value()` returns true. The + * value can be retrieved with `roaring64_iterator_value()`. + */ +roaring64_iterator_t *roaring64_iterator_create_last( + const roaring64_bitmap_t *r); + +/** + * Re-initializes an existing iterator. Functionally the same as + * `roaring64_iterator_create` without a allocation. + */ +void roaring64_iterator_reinit(const roaring64_bitmap_t *r, + roaring64_iterator_t *it); + +/** + * Re-initializes an existing iterator. Functionally the same as + * `roaring64_iterator_create_last` without a allocation. + */ +void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, + roaring64_iterator_t *it); + +/** + * Creates a copy of the iterator. Caller is responsible for calling + * `roaring64_iterator_free()` on the resulting iterator. + */ +roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it); + +/** + * Free the iterator. + */ +void roaring64_iterator_free(roaring64_iterator_t *it); + +/** + * Returns true if the iterator currently points to a value. If so, calling + * `roaring64_iterator_value()` returns the value. + */ +bool roaring64_iterator_has_value(const roaring64_iterator_t *it); + +/** + * Returns the value the iterator currently points to. Should only be called if + * `roaring64_iterator_has_value()` returns true. + */ +uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); + +/** + * Advance the iterator. If there is a new value, then + * `roaring64_iterator_has_value()` returns true. Values are traversed in + * increasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + * + * Once this returns false, `roaring64_iterator_advance` should not be called on + * the iterator again. Calling `roaring64_iterator_previous` is allowed. + */ +bool roaring64_iterator_advance(roaring64_iterator_t *it); + +/** + * Decrement the iterator. If there is a new value, then + * `roaring64_iterator_has_value()` returns true. Values are traversed in + * decreasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + * + * Once this returns false, `roaring64_iterator_previous` should not be called + * on the iterator again. Calling `roaring64_iterator_advance` is allowed. + */ +bool roaring64_iterator_previous(roaring64_iterator_t *it); + +/** + * Move the iterator to the first value greater than or equal to `val`, if it + * exists at or after the current position of the iterator. If there is a new + * value, then `roaring64_iterator_has_value()` returns true. Values are + * traversed in increasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + */ +bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, + uint64_t val); + +/** + * Reads up to `count` values from the iterator into the given `buf`. Returns + * the number of elements read. The number of elements read can be smaller than + * `count`, which means that there are no more elements in the bitmap. + * + * This function can be used together with other iterator functions. + */ +uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, + uint64_t count); + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace api +#endif + +#endif /* ROARING64_H */ +/* end file include/roaring/roaring64.h */ diff --git a/croaring-sys/CRoaring/roaring.hh b/croaring-sys/CRoaring/roaring.hh index afffef1..a8f4fad 100644 --- a/croaring-sys/CRoaring/roaring.hh +++ b/croaring-sys/CRoaring/roaring.hh @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2023-09-27T16:30:23Z +// Created by amalgamation.sh on 2024-04-02T13:42:32Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -64,9 +64,8 @@ A C++ header for Roaring Bitmaps. #ifndef INCLUDE_ROARING_HH_ #define INCLUDE_ROARING_HH_ -#include - #include +#include #include #include #include @@ -77,19 +76,19 @@ A C++ header for Roaring Bitmaps. #ifndef __cpp_exceptions #error "__cpp_exceptions should be defined" #endif -# if __cpp_exceptions -# define ROARING_EXCEPTIONS 1 -# else -# define ROARING_EXCEPTIONS 0 -# endif +#if __cpp_exceptions +#define ROARING_EXCEPTIONS 1 +#else +#define ROARING_EXCEPTIONS 0 +#endif #endif #ifndef ROARING_TERMINATE -# if ROARING_EXCEPTIONS -# define ROARING_TERMINATE(_s) throw std::runtime_error(_s) -# else -# define ROARING_TERMINATE(_s) std::terminate() -# endif +#if ROARING_EXCEPTIONS +#define ROARING_TERMINATE(_s) throw std::runtime_error(_s) +#else +#define ROARING_TERMINATE(_s) std::terminate() +#endif #endif #define ROARING_API_NOT_IN_GLOBAL_NAMESPACE // see remarks in roaring.h @@ -103,9 +102,9 @@ class RoaringSetBitForwardIterator; /** * A bit of context usable with `*Bulk()` functions. * - * A context may only be used with a single bitmap, and any modification to a bitmap - * (other than modifications performed with `Bulk()` functions with the context - * passed) will invalidate any contexts associated with that bitmap. + * A context may only be used with a single bitmap, and any modification to a + * bitmap (other than modifications performed with `Bulk()` functions with the + * context passed) will invalidate any contexts associated with that bitmap. */ class BulkContext { public: @@ -113,10 +112,10 @@ class BulkContext { using roaring_bitmap_bulk_context_t = api::roaring_bulk_context_t; BulkContext() : context_{nullptr, 0, 0, 0} {} - BulkContext(const BulkContext&) = delete; - BulkContext& operator=(const BulkContext&) = delete; - BulkContext(BulkContext&&) noexcept = default; - BulkContext& operator=(BulkContext&&) noexcept = default; + BulkContext(const BulkContext &) = delete; + BulkContext &operator=(const BulkContext &) = delete; + BulkContext(BulkContext &&) noexcept = default; + BulkContext &operator=(BulkContext &&) noexcept = default; private: roaring_bitmap_bulk_context_t context_; @@ -125,13 +124,14 @@ class BulkContext { class Roaring { typedef api::roaring_bitmap_t roaring_bitmap_t; // class-local name alias -public: + public: /** * Create an empty bitmap in the existing memory for the class. * The bitmap will be in the "clear" state with no auxiliary allocations. */ Roaring() : roaring{} { - // The empty constructor roaring{} silences warnings from pedantic static analyzers. + // The empty constructor roaring{} silences warnings from pedantic + // static analyzers. api::roaring_bitmap_init_cleared(&roaring); } @@ -158,8 +158,7 @@ public: ROARING_TERMINATE("failed roaring_bitmap_overwrite in constructor"); } api::roaring_bitmap_set_copy_on_write( - &roaring, - api::roaring_bitmap_get_copy_on_write(&r.roaring)); + &roaring, api::roaring_bitmap_get_copy_on_write(&r.roaring)); } /** @@ -183,7 +182,7 @@ public: * Passing a NULL pointer is unsafe. * The pointer to the C struct will be invalid after the call. */ - explicit Roaring(roaring_bitmap_t *s) noexcept : roaring (*s) { + explicit Roaring(roaring_bitmap_t *s) noexcept : roaring(*s) { roaring_free(s); // deallocate the passed-in pointer } @@ -259,21 +258,24 @@ public: } /** - * Check if item x is present, using context from a previous insert or search - * for speed optimization. + * Check if item x is present, using context from a previous insert or + * search for speed optimization. * * `context` will be used to store information between calls to make bulk * operations faster. `context` should be default-initialized before the * first call to this function. */ - bool containsBulk(BulkContext& context, uint32_t x) const noexcept { - return api::roaring_bitmap_contains_bulk(&roaring, &context.context_, x); + bool containsBulk(BulkContext &context, uint32_t x) const noexcept { + return api::roaring_bitmap_contains_bulk(&roaring, &context.context_, + x); } /** * Remove value x */ - void remove(uint32_t x) noexcept { api::roaring_bitmap_remove(&roaring, x); } + void remove(uint32_t x) noexcept { + api::roaring_bitmap_remove(&roaring, x); + } /** * Remove value x @@ -301,12 +303,16 @@ public: /** * Return the largest value (if not empty) */ - uint32_t maximum() const noexcept { return api::roaring_bitmap_maximum(&roaring); } + uint32_t maximum() const noexcept { + return api::roaring_bitmap_maximum(&roaring); + } /** * Return the smallest value (if not empty) */ - uint32_t minimum() const noexcept { return api::roaring_bitmap_minimum(&roaring); } + uint32_t minimum() const noexcept { + return api::roaring_bitmap_minimum(&roaring); + } /** * Check if value x is present @@ -338,7 +344,7 @@ public: // `roaring_bitmap_frozen_view` and free it as well. roaring_bitmap_free( (roaring_bitmap_t *)((char *) - roaring.high_low_container.containers - + roaring.high_low_container.containers - sizeof(roaring_bitmap_t))); } } @@ -353,8 +359,7 @@ public: ROARING_TERMINATE("failed memory alloc in assignment"); } api::roaring_bitmap_set_copy_on_write( - &roaring, - api::roaring_bitmap_get_copy_on_write(&r.roaring)); + &roaring, api::roaring_bitmap_get_copy_on_write(&r.roaring)); return *this; } @@ -442,7 +447,9 @@ public: /** * Returns true if the bitmap is empty (cardinality is zero). */ - bool isEmpty() const noexcept { return api::roaring_bitmap_is_empty(&roaring); } + bool isEmpty() const noexcept { + return api::roaring_bitmap_is_empty(&roaring); + } /** * Returns true if the bitmap is subset of the other. @@ -469,7 +476,8 @@ public: /** * To int array with pagination */ - void rangeUint32Array(uint32_t *ans, size_t offset, size_t limit) const noexcept { + void rangeUint32Array(uint32_t *ans, size_t offset, + size_t limit) const noexcept { api::roaring_bitmap_range_uint32_array(&roaring, offset, limit, ans); } @@ -493,8 +501,8 @@ public: * [range_start, range_end]. Areas outside the interval are unchanged. */ void flipClosed(uint32_t range_start, uint32_t range_end) noexcept { - api::roaring_bitmap_flip_inplace( - &roaring, range_start, uint64_t(range_end) + 1); + api::roaring_bitmap_flip_inplace(&roaring, range_start, + uint64_t(range_end) + 1); } /** @@ -511,13 +519,17 @@ public: * Returns true if the result has at least one run container. Additional * savings might be possible by calling shrinkToFit(). */ - bool runOptimize() noexcept { return api::roaring_bitmap_run_optimize(&roaring); } + bool runOptimize() noexcept { + return api::roaring_bitmap_run_optimize(&roaring); + } /** * If needed, reallocate memory to shrink the memory usage. Returns * the number of bytes saved. */ - size_t shrinkToFit() noexcept { return api::roaring_bitmap_shrink_to_fit(&roaring); } + size_t shrinkToFit() noexcept { + return api::roaring_bitmap_shrink_to_fit(&roaring); + } /** * Iterate over the bitmap elements. The function iterator is called once @@ -603,6 +615,16 @@ public: return api::roaring_bitmap_rank(&roaring, x); } + /** + * Get `rank()` values in bulk. The values in `[begin .. end)` must be in + * Ascending order. possible implementation: for(auto* iter = begin; iter != + * end; ++iter) *(ans++) = rank(*iter); + */ + void rank_many(const uint32_t *begin, const uint32_t *end, + uint64_t *ans) const noexcept { + return api::roaring_bitmap_rank_many(&roaring, begin, end, ans); + } + /** * Returns the index of x in the set, index start from 0. * If the set doesn't contain x , this function will return -1. @@ -672,15 +694,15 @@ public: * This function is unsafe in the sense that if you provide bad data, * many, many bytes could be read. See also readSafe. * - * The function may throw std::runtime_error if a bitmap could not be read. Not that even - * if it does not throw, the bitmap could still be unusable if the loaded - * data does not match the portable Roaring specification: you should - * ensure that the data you load come from a serialized bitmap. + * The function may throw std::runtime_error if a bitmap could not be read. + * Not that even if it does not throw, the bitmap could still be unusable if + * the loaded data does not match the portable Roaring specification: you + * should ensure that the data you load come from a serialized bitmap. */ static Roaring read(const char *buf, bool portable = true) { - roaring_bitmap_t * r = portable - ? api::roaring_bitmap_portable_deserialize(buf) - : api::roaring_bitmap_deserialize(buf); + roaring_bitmap_t *r = + portable ? api::roaring_bitmap_portable_deserialize(buf) + : api::roaring_bitmap_deserialize(buf); if (r == NULL) { ROARING_TERMINATE("failed alloc while reading"); } @@ -690,27 +712,29 @@ public: /** * Read a bitmap from a serialized version, reading no more than maxbytes * bytes. This is meant to be compatible with the Java and Go versions. - * The function itself is safe in the sense that it will not cause buffer overflows. - * However, for correct operations, it is assumed that the bitmap read was once - * serialized from a valid bitmap. If you provided an incorrect input (garbage), then the - * bitmap read may not be in a valid state and following operations may not lead - * to sensible results. It is your responsability to ensure that the input bytes - * follow the format specification if you want a usable bitmap: + * The function itself is safe in the sense that it will not cause buffer + * overflows. However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap. If you provided an + * incorrect input (garbage), then the bitmap read may not be in a valid + * state and following operations may not lead to sensible results. It is + * your responsability to ensure that the input bytes follow the format + * specification if you want a usable bitmap: * https://github.com/RoaringBitmap/RoaringFormatSpec - * In particular, the serialized array containers need to be in sorted order, and the - * run containers should be in sorted non-overlapping order. This is is guaranteed to - * happen when serializing an existing bitmap, but not for random inputs. - * Note that this function assumes that your bitmap was serialized in *portable* mode - * (which is the default with the 'write' method). + * In particular, the serialized array containers need to be in sorted + * order, and the run containers should be in sorted non-overlapping order. + * This is is guaranteed to happen when serializing an existing bitmap, but + * not for random inputs. Note that this function assumes that your bitmap + * was serialized in *portable* mode (which is the default with the 'write' + * method). * - * The function may throw std::runtime_error if a bitmap could not be read. Not that even - * if it does not throw, the bitmap could still be unusable if the loaded - * data does not match the portable Roaring specification: you should - * ensure that the data you load come from a serialized bitmap. + * The function may throw std::runtime_error if a bitmap could not be read. + * Not that even if it does not throw, the bitmap could still be unusable if + * the loaded data does not match the portable Roaring specification: you + * should ensure that the data you load come from a serialized bitmap. */ static Roaring readSafe(const char *buf, size_t maxbytes) { - roaring_bitmap_t * r = - api::roaring_bitmap_portable_deserialize_safe(buf,maxbytes); + roaring_bitmap_t *r = + api::roaring_bitmap_portable_deserialize_safe(buf, maxbytes); if (r == NULL) { ROARING_TERMINATE("failed alloc while reading"); } @@ -748,6 +772,21 @@ public: return r; } + /** + * For advanced users; see roaring_bitmap_portable_deserialize_frozen. + * This function may throw std::runtime_error. + */ + static const Roaring portableDeserializeFrozen(const char *buf) { + const roaring_bitmap_t *s = + api::roaring_bitmap_portable_deserialize_frozen(buf); + if (s == NULL) { + ROARING_TERMINATE("failed to read portable frozen bitmap"); + } + Roaring r; + r.roaring = *s; + return r; + } + /** * For advanced users. */ @@ -836,7 +875,8 @@ public: */ std::string toString() const noexcept { struct iter_data { - std::string str{}; // The empty constructor silences warnings from pedantic static analyzers. + std::string str{}; // The empty constructor silences warnings from + // pedantic static analyzers. char first_char = '{'; } outer_iter_data; if (!isEmpty()) { @@ -869,8 +909,8 @@ public: * This function may throw std::runtime_error. */ static Roaring fastunion(size_t n, const Roaring **inputs) { - const roaring_bitmap_t **x = - (const roaring_bitmap_t **)roaring_malloc(n * sizeof(roaring_bitmap_t *)); + const roaring_bitmap_t **x = (const roaring_bitmap_t **)roaring_malloc( + n * sizeof(roaring_bitmap_t *)); if (x == NULL) { ROARING_TERMINATE("failed memory alloc in fastunion"); } @@ -912,7 +952,7 @@ public: * Used to go through the set bits. Not optimally fast, but convenient. */ class RoaringSetBitForwardIterator final { -public: + public: typedef std::forward_iterator_tag iterator_category; typedef uint32_t *pointer; typedef uint32_t &reference_type; @@ -937,13 +977,13 @@ public: return i.current_value <= *o; } - bool operator>(const type_of_iterator &o) const { + bool operator>(const type_of_iterator &o) const { if (!o.i.has_value) return false; if (!i.has_value) return true; return i.current_value > *o; } - bool operator>=(const type_of_iterator &o) const { + bool operator>=(const type_of_iterator &o) const { if (!i.has_value) return true; if (!o.i.has_value) return false; return i.current_value >= *o; @@ -953,28 +993,28 @@ public: * Move the iterator to the first value >= val. */ void equalorlarger(uint32_t val) { - api::roaring_move_uint32_iterator_equalorlarger(&i,val); + api::roaring_uint32_iterator_move_equalorlarger(&i, val); } type_of_iterator &operator++() { // ++i, must returned inc. value - api::roaring_advance_uint32_iterator(&i); + api::roaring_uint32_iterator_advance(&i); return *this; } type_of_iterator operator++(int) { // i++, must return orig. value RoaringSetBitForwardIterator orig(*this); - api::roaring_advance_uint32_iterator(&i); + api::roaring_uint32_iterator_advance(&i); return orig; } - type_of_iterator& operator--() { // prefix -- - api::roaring_previous_uint32_iterator(&i); + type_of_iterator &operator--() { // prefix -- + api::roaring_uint32_iterator_previous(&i); return *this; } - type_of_iterator operator--(int) { // postfix -- + type_of_iterator operator--(int) { // postfix -- RoaringSetBitForwardIterator orig(*this); - api::roaring_previous_uint32_iterator(&i); + api::roaring_uint32_iterator_previous(&i); return orig; } @@ -994,11 +1034,13 @@ public: i.has_value = false; i.current_value = UINT32_MAX; } else { - api::roaring_init_iterator(&parent.roaring, &i); + api::roaring_iterator_init(&parent.roaring, &i); } } - api::roaring_uint32_iterator_t i{}; // The empty constructor silences warnings from pedantic static analyzers. + api::roaring_uint32_iterator_t + i{}; // The empty constructor silences warnings from pedantic static + // analyzers. }; inline RoaringSetBitForwardIterator Roaring::begin() const { @@ -1016,21 +1058,21 @@ inline RoaringSetBitForwardIterator &Roaring::end() const { /* end file cpp/roaring.hh */ /* begin file cpp/roaring64map.hh */ /** - * A C++ header for 64-bit Roaring Bitmaps, + * A C++ header for 64-bit Roaring Bitmaps, * implemented by way of a map of many * 32-bit Roaring Bitmaps. - * + * * Reference (format specification) : * https://github.com/RoaringBitmap/RoaringFormatSpec#extention-for-64-bit-implementations -*/ + */ #ifndef INCLUDE_ROARING_64_MAP_HH_ #define INCLUDE_ROARING_64_MAP_HH_ #include -#include // PRIu64 macro -#include // for va_list handling in bitmapOf() -#include // for std::printf() in the printf() method -#include // for std::memcpy() +#include // PRIu64 macro +#include // for va_list handling in bitmapOf() +#include // for std::printf() in the printf() method +#include // for std::memcpy() #include #include #include @@ -1047,13 +1089,17 @@ namespace roaring { using roaring::Roaring; -class Roaring64MapSetBitForwardIterator; class Roaring64MapSetBitBiDirectionalIterator; +// For backwards compatibility; there used to be two kinds of iterators +// (forward and bidirectional) and now there's only one. +typedef Roaring64MapSetBitBiDirectionalIterator + Roaring64MapSetBitForwardIterator; + class Roaring64Map { typedef api::roaring_bitmap_t roaring_bitmap_t; -public: + public: /** * Create an empty bitmap */ @@ -1095,9 +1141,9 @@ public: emplaceOrInsert(0, Roaring(s)); } - Roaring64Map(const Roaring64Map& r) = default; + Roaring64Map(const Roaring64Map &r) = default; - Roaring64Map(Roaring64Map&& r) noexcept = default; + Roaring64Map(Roaring64Map &&r) noexcept = default; /** * Copy assignment operator. @@ -1145,25 +1191,19 @@ public: /** * Adds value x. */ - void add(uint32_t x) { - lookupOrCreateInner(0).add(x); - } + void add(uint32_t x) { lookupOrCreateInner(0).add(x); } /** * Adds value x. */ - void add(uint64_t x) { - lookupOrCreateInner(highBytes(x)).add(lowBytes(x)); - } + void add(uint64_t x) { lookupOrCreateInner(highBytes(x)).add(lowBytes(x)); } /** * Adds value x. * Returns true if a new value was added, false if the value was already * present. */ - bool addChecked(uint32_t x) { - return lookupOrCreateInner(0).addChecked(x); - } + bool addChecked(uint32_t x) { return lookupOrCreateInner(0).addChecked(x); } /** * Adds value x. @@ -1270,6 +1310,7 @@ public: // assuming that adjacent values will belong to the same inner bitmap. Roaring *last_inner_bitmap = nullptr; uint32_t last_value_high = 0; + BulkContext last_bulk_context; for (size_t lcv = 0; lcv < n_args; lcv++) { auto value = vals[lcv]; auto value_high = highBytes(value); @@ -1277,8 +1318,9 @@ public: if (last_inner_bitmap == nullptr || value_high != last_value_high) { last_inner_bitmap = &lookupOrCreateInner(value_high); last_value_high = value_high; + last_bulk_context = BulkContext{}; } - last_inner_bitmap->add(value_low); + last_inner_bitmap->addBulk(last_bulk_context, value_low); } } @@ -1418,7 +1460,8 @@ public: // a. if the end point falls on that same entry, remove the closed // interval [start_low, end_low] from that entry and we are done. // b. Otherwise, remove the closed interval [start_low, uint32_max] - // from that entry, advance start_iter, and fall through to step 2. + // from that entry, advance start_iter, and fall through to + // step 2. // 2. Completely erase all slots in the half-open interval // [start_iter, end_iter) // 3. If the end point falls on an existing entry, remove the closed @@ -1456,9 +1499,7 @@ public: /** * Clears the bitmap. */ - void clear() { - roarings.clear(); - } + void clear() { roarings.clear(); } /** * Return the largest value (if not empty) @@ -1496,12 +1537,18 @@ public: * Check if value x is present */ bool contains(uint32_t x) const { - return roarings.count(0) == 0 ? false : roarings.at(0).contains(x); + auto iter = roarings.find(0); + if (iter == roarings.end()) { + return false; + } + return iter->second.contains(x); } bool contains(uint64_t x) const { - return roarings.count(highBytes(x)) == 0 - ? false - : roarings.at(highBytes(x)).contains(lowBytes(x)); + auto iter = roarings.find(highBytes(x)); + if (iter == roarings.end()) { + return false; + } + return iter->second.contains(lowBytes(x)); } /** @@ -1758,11 +1805,13 @@ public: uint64_t cardinality() const { if (isFull()) { #if ROARING_EXCEPTIONS - throw std::length_error("bitmap is full, cardinality is 2^64, " - "unable to represent in a 64-bit integer"); + throw std::length_error( + "bitmap is full, cardinality is 2^64, " + "unable to represent in a 64-bit integer"); #else - ROARING_TERMINATE("bitmap is full, cardinality is 2^64, " - "unable to represent in a 64-bit integer"); + ROARING_TERMINATE( + "bitmap is full, cardinality is 2^64, " + "unable to represent in a 64-bit integer"); #endif } return std::accumulate( @@ -1777,10 +1826,11 @@ public: * Returns true if the bitmap is empty (cardinality is zero). */ bool isEmpty() const { - return std::all_of(roarings.cbegin(), roarings.cend(), - [](const std::pair &map_entry) { - return map_entry.second.isEmpty(); - }); + return std::all_of( + roarings.cbegin(), roarings.cend(), + [](const std::pair &map_entry) { + return map_entry.second.isEmpty(); + }); } /** @@ -1792,18 +1842,19 @@ public: // we put std::numeric_limits<>::max/min in parentheses // to avoid a clash with the Windows.h header under Windows return roarings.size() == - ((uint64_t)(std::numeric_limits::max)()) + 1 - ? std::all_of( - roarings.cbegin(), roarings.cend(), - [](const std::pair &roaring_map_entry) { - // roarings within map are saturated if cardinality - // is uint32_t max + 1 - return roaring_map_entry.second.cardinality() == - ((uint64_t) - (std::numeric_limits::max)()) + - 1; - }) - : false; + ((uint64_t)(std::numeric_limits::max)()) + 1 + ? std::all_of( + roarings.cbegin(), roarings.cend(), + [](const std::pair + &roaring_map_entry) { + // roarings within map are saturated if cardinality + // is uint32_t max + 1 + return roaring_map_entry.second.cardinality() == + ((uint64_t)(std::numeric_limits< + uint32_t>::max)()) + + 1; + }) + : false; } /** @@ -1826,7 +1877,8 @@ public: /** * Returns true if the bitmap is strict subset of the other. * Throws std::length_error in the special case where the bitmap is full - * (cardinality() == 2^64). Check isFull() before calling to avoid exception. + * (cardinality() == 2^64). Check isFull() before calling to avoid + * exception. */ bool isStrictSubset(const Roaring64Map &r) const { return isSubset(r) && cardinality() != r.cardinality(); @@ -1840,14 +1892,14 @@ public: */ void toUint64Array(uint64_t *ans) const { // Annoyingly, VS 2017 marks std::accumulate() as [[nodiscard]] - (void)std::accumulate(roarings.cbegin(), roarings.cend(), ans, - [](uint64_t *previous, - const std::pair &map_entry) { - for (uint32_t low_bits : map_entry.second) - *previous++ = - uniteBytes(map_entry.first, low_bits); - return previous; - }); + (void)std::accumulate( + roarings.cbegin(), roarings.cend(), ans, + [](uint64_t *previous, + const std::pair &map_entry) { + for (uint32_t low_bits : map_entry.second) + *previous++ = uniteBytes(map_entry.first, low_bits); + return previous; + }); } /** @@ -1933,7 +1985,7 @@ public: */ void flipClosed(uint64_t min, uint64_t max) { if (min > max) { - return; + return; } uint32_t start_high = highBytes(min); uint32_t start_low = lowBytes(min); @@ -2075,7 +2127,8 @@ public: // Casting rank to uint32_t is safe because // rank < sub_cardinality and sub_cardinality <= 2^32. if (!bitmap.select((uint32_t)rank, &low_bytes)) { - ROARING_TERMINATE("Logic error: bitmap.select() " + ROARING_TERMINATE( + "Logic error: bitmap.select() " "returned false despite rank < cardinality()"); } *element = uniteBytes(key, low_bytes); @@ -2091,8 +2144,9 @@ public: */ uint64_t rank(uint64_t x) const { uint64_t result = 0; - // Find the first bitmap >= x's bucket. If that is the bucket x would be in, find it's rank in that bucket. - // Either way, we're left with a range of all buckets strictly smaller than x's bucket, add all their + // Find the first bitmap >= x's bucket. If that is the bucket x would be + // in, find it's rank in that bucket. Either way, we're left with a + // range of all buckets strictly smaller than x's bucket, add all their // cardinalities together. auto end = roarings.lower_bound(highBytes(x)); if (end != roarings.cend() && end->first == highBytes(x)) { @@ -2142,17 +2196,18 @@ public: uint64_t map_size = roarings.size(); std::memcpy(buf, &map_size, sizeof(uint64_t)); buf += sizeof(uint64_t); - std::for_each( - roarings.cbegin(), roarings.cend(), - [&buf, portable](const std::pair &map_entry) { - // push map key - std::memcpy(buf, &map_entry.first, sizeof(uint32_t)); - // ^-- Note: `*((uint32_t*)buf) = map_entry.first;` is undefined - - buf += sizeof(uint32_t); - // push map value Roaring - buf += map_entry.second.write(buf, portable); - }); + std::for_each(roarings.cbegin(), roarings.cend(), + [&buf, portable]( + const std::pair &map_entry) { + // push map key + std::memcpy(buf, &map_entry.first, sizeof(uint32_t)); + // ^-- Note: `*((uint32_t*)buf) = map_entry.first;` is + // undefined + + buf += sizeof(uint32_t); + // push map value Roaring + buf += map_entry.second.write(buf, portable); + }); return buf - orig; } @@ -2207,7 +2262,7 @@ public: buf += sizeof(uint64_t); maxbytes -= sizeof(uint64_t); for (uint64_t lcv = 0; lcv < map_size; lcv++) { - if(maxbytes < sizeof(uint32_t)) { + if (maxbytes < sizeof(uint32_t)) { ROARING_TERMINATE("ran out of bytes"); } uint32_t key; @@ -2282,6 +2337,26 @@ public: return result; } + static const Roaring64Map portableDeserializeFrozen(const char *buf) { + Roaring64Map result; + // get map size + uint64_t map_size; + std::memcpy(&map_size, buf, sizeof(uint64_t)); + buf += sizeof(uint64_t); + for (uint64_t lcv = 0; lcv < map_size; lcv++) { + // get map key + uint32_t key; + std::memcpy(&key, buf, sizeof(uint32_t)); + buf += sizeof(uint32_t); + // read map value Roaring + Roaring read_var = Roaring::portableDeserializeFrozen(buf); + // forward buffer past the last Roaring bitmap + buf += read_var.getSizeInBytes(true); + result.emplaceOrInsert(key, std::move(read_var)); + } + return result; + } + // As with serialized 64-bit bitmaps, 64-bit frozen bitmaps are serialized // by concatenating one or more Roaring::write output buffers with the // preceeding map key. Unlike standard bitmap serialization, frozen bitmaps @@ -2393,9 +2468,7 @@ public: * Note: this method adds a final newline, but toString() does not. */ void printf() const { - auto sink = [](const std::string &s) { - fputs(s.c_str(), stdout); - }; + auto sink = [](const std::string &s) { fputs(s.c_str(), stdout); }; printToSink(sink); sink("\n"); } @@ -2405,9 +2478,7 @@ public: */ std::string toString() const { std::string result; - auto sink = [&result](const std::string &s) { - result += s; - }; + auto sink = [&result](const std::string &s) { result += s; }; printToSink(sink); return result; } @@ -2457,7 +2528,8 @@ public: }; // Create and populate the priority queue. - std::priority_queue, decltype(pq_comp)> pq(pq_comp); + std::priority_queue, decltype(pq_comp)> + pq(pq_comp); for (size_t i = 0; i < n; ++i) { const auto &roarings = inputs[i]->roarings; if (roarings.begin() != roarings.end()) { @@ -2467,7 +2539,7 @@ public: // A reusable vector that holds the pointers to the inner bitmaps that // we pass to the underlying 32-bit fastunion operation. - std::vector group_bitmaps; + std::vector group_bitmaps; // Summary of the algorithm: // 1. While the priority queue is not empty: @@ -2533,19 +2605,20 @@ public: // Use the fast inner union to combine these. auto *inner_result = roaring_bitmap_or_many(group_bitmaps.size(), - group_bitmaps.data()); + group_bitmaps.data()); // Insert the 32-bit result at end of the 'roarings' map of the // result we are building. - result.roarings.insert(result.roarings.end(), + result.roarings.insert( + result.roarings.end(), std::make_pair(group_key, Roaring(inner_result))); } return result; } - friend class Roaring64MapSetBitForwardIterator; friend class Roaring64MapSetBitBiDirectionalIterator; - typedef Roaring64MapSetBitForwardIterator const_iterator; - typedef Roaring64MapSetBitBiDirectionalIterator const_bidirectional_iterator; + typedef Roaring64MapSetBitBiDirectionalIterator const_iterator; + typedef Roaring64MapSetBitBiDirectionalIterator + const_bidirectional_iterator; /** * Returns an iterator that can be used to access the position of the set @@ -2564,12 +2637,17 @@ public: */ const_iterator end() const; -private: + private: typedef std::map roarings_t; - roarings_t roarings{}; // The empty constructor silences warnings from pedantic static analyzers. + roarings_t roarings{}; // The empty constructor silences warnings from + // pedantic static analyzers. bool copyOnWrite{false}; - static constexpr uint32_t highBytes(const uint64_t in) { return uint32_t(in >> 32); } - static constexpr uint32_t lowBytes(const uint64_t in) { return uint32_t(in); } + static constexpr uint32_t highBytes(const uint64_t in) { + return uint32_t(in >> 32); + } + static constexpr uint32_t lowBytes(const uint64_t in) { + return uint32_t(in); + } static constexpr uint64_t uniteBytes(const uint32_t highBytes, const uint32_t lowBytes) { return (uint64_t(highBytes) << 32) | uint64_t(lowBytes); @@ -2606,7 +2684,8 @@ private: /** * Prints the contents of the bitmap to a caller-provided sink function. */ - void printToSink(const std::function &sink) const { + void printToSink( + const std::function &sink) const { sink("{"); // Storage for snprintf. Big enough to store the decimal representation @@ -2692,14 +2771,30 @@ private: /** * Used to go through the set bits. Not optimally fast, but convenient. */ -class Roaring64MapSetBitForwardIterator { -public: - typedef std::forward_iterator_tag iterator_category; +class Roaring64MapSetBitBiDirectionalIterator { + public: + typedef std::bidirectional_iterator_tag iterator_category; typedef uint64_t *pointer; typedef uint64_t &reference; typedef uint64_t value_type; typedef int64_t difference_type; - typedef Roaring64MapSetBitForwardIterator type_of_iterator; + typedef Roaring64MapSetBitBiDirectionalIterator type_of_iterator; + + Roaring64MapSetBitBiDirectionalIterator(const Roaring64Map &parent, + bool exhausted = false) + : p(&parent.roarings) { + if (exhausted || parent.roarings.empty()) { + map_iter = p->cend(); + } else { + map_iter = parent.roarings.cbegin(); + roaring_iterator_init(&map_iter->second.roaring, &i); + while (!i.has_value) { + map_iter++; + if (map_iter == p->cend()) return; + roaring_iterator_init(&map_iter->second.roaring, &i); + } + } + } /** * Provides the location of the set bit. @@ -2709,170 +2804,128 @@ public: } bool operator<(const type_of_iterator &o) const { - if (map_iter == map_end) return false; - if (o.map_iter == o.map_end) return true; + if (map_iter == p->cend()) return false; + if (o.map_iter == o.p->cend()) return true; return **this < *o; } bool operator<=(const type_of_iterator &o) const { - if (o.map_iter == o.map_end) return true; - if (map_iter == map_end) return false; + if (o.map_iter == o.p->cend()) return true; + if (map_iter == p->cend()) return false; return **this <= *o; } bool operator>(const type_of_iterator &o) const { - if (o.map_iter == o.map_end) return false; - if (map_iter == map_end) return true; + if (o.map_iter == o.p->cend()) return false; + if (map_iter == p->cend()) return true; return **this > *o; } bool operator>=(const type_of_iterator &o) const { - if (map_iter == map_end) return true; - if (o.map_iter == o.map_end) return false; + if (map_iter == p->cend()) return true; + if (o.map_iter == o.p->cend()) return false; return **this >= *o; } type_of_iterator &operator++() { // ++i, must returned inc. value - if (i.has_value == true) roaring_advance_uint32_iterator(&i); + if (i.has_value == true) roaring_uint32_iterator_advance(&i); while (!i.has_value) { - map_iter++; - if (map_iter == map_end) return *this; - roaring_init_iterator(&map_iter->second.roaring, &i); + ++map_iter; + if (map_iter == p->cend()) return *this; + roaring_iterator_init(&map_iter->second.roaring, &i); } return *this; } type_of_iterator operator++(int) { // i++, must return orig. value - Roaring64MapSetBitForwardIterator orig(*this); - roaring_advance_uint32_iterator(&i); + Roaring64MapSetBitBiDirectionalIterator orig(*this); + roaring_uint32_iterator_advance(&i); while (!i.has_value) { - map_iter++; - if (map_iter == map_end) return orig; - roaring_init_iterator(&map_iter->second.roaring, &i); + ++map_iter; + if (map_iter == p->cend()) return orig; + roaring_iterator_init(&map_iter->second.roaring, &i); } return orig; } - bool move(const value_type& x) { - map_iter = p.lower_bound(Roaring64Map::highBytes(x)); - if (map_iter != p.cend()) { - roaring_init_iterator(&map_iter->second.roaring, &i); + bool move(const value_type &x) { + map_iter = p->lower_bound(Roaring64Map::highBytes(x)); + if (map_iter != p->cend()) { + roaring_iterator_init(&map_iter->second.roaring, &i); if (map_iter->first == Roaring64Map::highBytes(x)) { - if (roaring_move_uint32_iterator_equalorlarger(&i, Roaring64Map::lowBytes(x))) + if (roaring_uint32_iterator_move_equalorlarger( + &i, Roaring64Map::lowBytes(x))) return true; - map_iter++; - if (map_iter == map_end) return false; - roaring_init_iterator(&map_iter->second.roaring, &i); + ++map_iter; + if (map_iter == p->cend()) return false; + roaring_iterator_init(&map_iter->second.roaring, &i); } return true; } return false; } - bool operator==(const Roaring64MapSetBitForwardIterator &o) const { - if (map_iter == map_end && o.map_iter == o.map_end) return true; - if (o.map_iter == o.map_end) return false; - return **this == *o; - } - - bool operator!=(const Roaring64MapSetBitForwardIterator &o) const { - if (map_iter == map_end && o.map_iter == o.map_end) return false; - if (o.map_iter == o.map_end) return true; - return **this != *o; - } - - Roaring64MapSetBitForwardIterator &operator=(const Roaring64MapSetBitForwardIterator& r) { - map_iter = r.map_iter; - map_end = r.map_end; - i = r.i; - return *this; - } - - Roaring64MapSetBitForwardIterator(const Roaring64MapSetBitForwardIterator& r) - : p(r.p), - map_iter(r.map_iter), - map_end(r.map_end), - i(r.i) - {} - - Roaring64MapSetBitForwardIterator(const Roaring64Map &parent, - bool exhausted = false) - : p(parent.roarings), map_end(parent.roarings.cend()) { - if (exhausted || parent.roarings.empty()) { - map_iter = parent.roarings.cend(); - } else { - map_iter = parent.roarings.cbegin(); - roaring_init_iterator(&map_iter->second.roaring, &i); - while (!i.has_value) { - map_iter++; - if (map_iter == map_end) return; - roaring_init_iterator(&map_iter->second.roaring, &i); - } - } - } - -protected: - const std::map& p; - std::map::const_iterator map_iter{}; // The empty constructor silences warnings from pedantic static analyzers. - std::map::const_iterator map_end{}; // The empty constructor silences warnings from pedantic static analyzers. - api::roaring_uint32_iterator_t i{}; // The empty constructor silences warnings from pedantic static analyzers. -}; - -class Roaring64MapSetBitBiDirectionalIterator final :public Roaring64MapSetBitForwardIterator { -public: - explicit Roaring64MapSetBitBiDirectionalIterator(const Roaring64Map &parent, - bool exhausted = false) - : Roaring64MapSetBitForwardIterator(parent, exhausted), map_begin(parent.roarings.cbegin()) - {} - - Roaring64MapSetBitBiDirectionalIterator &operator=(const Roaring64MapSetBitForwardIterator& r) { - *(Roaring64MapSetBitForwardIterator*)this = r; - return *this; - } - - Roaring64MapSetBitBiDirectionalIterator& operator--() { // --i, must return dec.value - if (map_iter == map_end) { + type_of_iterator &operator--() { // --i, must return dec.value + if (map_iter == p->cend()) { --map_iter; - roaring_init_iterator_last(&map_iter->second.roaring, &i); + roaring_iterator_init_last(&map_iter->second.roaring, &i); if (i.has_value) return *this; } - roaring_previous_uint32_iterator(&i); + roaring_uint32_iterator_previous(&i); while (!i.has_value) { - if (map_iter == map_begin) return *this; + if (map_iter == p->cbegin()) return *this; map_iter--; - roaring_init_iterator_last(&map_iter->second.roaring, &i); + roaring_iterator_init_last(&map_iter->second.roaring, &i); } return *this; } - Roaring64MapSetBitBiDirectionalIterator operator--(int) { // i--, must return orig. value + type_of_iterator operator--(int) { // i--, must return orig. value Roaring64MapSetBitBiDirectionalIterator orig(*this); - if (map_iter == map_end) { + if (map_iter == p->cend()) { --map_iter; - roaring_init_iterator_last(&map_iter->second.roaring, &i); + roaring_iterator_init_last(&map_iter->second.roaring, &i); return orig; } - roaring_previous_uint32_iterator(&i); + roaring_uint32_iterator_previous(&i); while (!i.has_value) { - if (map_iter == map_begin) return orig; + if (map_iter == p->cbegin()) return orig; map_iter--; - roaring_init_iterator_last(&map_iter->second.roaring, &i); + roaring_iterator_init_last(&map_iter->second.roaring, &i); } return orig; } -protected: - std::map::const_iterator map_begin; + bool operator==(const Roaring64MapSetBitBiDirectionalIterator &o) const { + if (map_iter == p->cend() && o.map_iter == o.p->cend()) return true; + if (o.map_iter == o.p->cend()) return false; + return **this == *o; + } + + bool operator!=(const Roaring64MapSetBitBiDirectionalIterator &o) const { + if (map_iter == p->cend() && o.map_iter == o.p->cend()) return false; + if (o.map_iter == o.p->cend()) return true; + return **this != *o; + } + + private: + const std::map *p{nullptr}; + std::map::const_iterator + map_iter{}; // The empty constructor silences warnings from pedantic + // static analyzers. + api::roaring_uint32_iterator_t + i{}; // The empty constructor silences warnings from pedantic static + // analyzers. }; -inline Roaring64MapSetBitForwardIterator Roaring64Map::begin() const { - return Roaring64MapSetBitForwardIterator(*this); +inline Roaring64MapSetBitBiDirectionalIterator Roaring64Map::begin() const { + return Roaring64MapSetBitBiDirectionalIterator(*this); } -inline Roaring64MapSetBitForwardIterator Roaring64Map::end() const { - return Roaring64MapSetBitForwardIterator(*this, true); +inline Roaring64MapSetBitBiDirectionalIterator Roaring64Map::end() const { + return Roaring64MapSetBitBiDirectionalIterator(*this, true); } } // namespace roaring diff --git a/croaring-sys/Cargo.toml b/croaring-sys/Cargo.toml index 8f4b5a5..04af8e4 100644 --- a/croaring-sys/Cargo.toml +++ b/croaring-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "croaring-sys" -version = "1.1.0" +version = "2.0.0" edition = "2021" authors = ["croaring-rs developers"] license = "Apache-2.0" diff --git a/croaring-sys/build.rs b/croaring-sys/build.rs index b76e896..f37ac53 100644 --- a/croaring-sys/build.rs +++ b/croaring-sys/build.rs @@ -1,5 +1,4 @@ use std::env; -use std::path::PathBuf; fn main() { println!("cargo:rerun-if-changed=CRoaring"); diff --git a/croaring/Cargo.toml b/croaring/Cargo.toml index 3eea104..060fc0c 100644 --- a/croaring/Cargo.toml +++ b/croaring/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "croaring" -version = "1.0.1" +version = "1.1.0" edition = "2021" authors = ["croaring-rs developers"] license = "Apache-2.0" @@ -22,7 +22,7 @@ roaring = "0.10" criterion = { version = "0.5", features = ["html_reports"] } [dependencies] -ffi = { package = "croaring-sys", path = "../croaring-sys", version = "1.1.0" } +ffi = { package = "croaring-sys", path = "../croaring-sys", version = "2.0.0" } byteorder = "1.4.3" [[bench]] diff --git a/croaring/benches/benches.rs b/croaring/benches/benches.rs index 6bfba71..e362f33 100644 --- a/croaring/benches/benches.rs +++ b/croaring/benches/benches.rs @@ -1,8 +1,9 @@ use criterion::{ black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput, }; +use std::ops::ControlFlow; -use croaring::{Bitmap, Portable}; +use croaring::{Bitmap, Bitmap64, Portable}; fn new(c: &mut Criterion) { c.bench_function("new", |b| b.iter(Bitmap::new)); @@ -130,10 +131,25 @@ fn flip(c: &mut Criterion) { } fn to_vec(c: &mut Criterion) { - c.bench_function("to_vec", |b| { - let bitmap = Bitmap::of(&[1, 2, 3]); + const N: usize = 100_000; + let bitmap: Bitmap = random_iter().take(N).collect(); + let mut g = c.benchmark_group("collect"); + g.bench_function("to_vec", |b| { b.iter(|| bitmap.to_vec()); }); + g.bench_function("via_iter", |b| { + b.iter(|| bitmap.iter().collect::>()); + }); + g.bench_function("foreach", |b| { + b.iter(|| { + let mut vec = Vec::with_capacity(bitmap.cardinality() as usize); + bitmap.for_each(|item| -> ControlFlow<()> { + vec.push(item); + ControlFlow::Continue(()) + }); + vec + }); + }); } fn get_serialized_size_in_bytes(c: &mut Criterion) { @@ -213,7 +229,27 @@ fn bulk_new(c: &mut Criterion) { group.finish(); } -fn random_iter(c: &mut Criterion) { +#[derive(Clone)] +struct RandomIter { + x: u32, +} + +impl Iterator for RandomIter { + type Item = u32; + + fn next(&mut self) -> Option { + const MULTIPLIER: u32 = 742938285; + const MODULUS: u32 = (1 << 31) - 1; + self.x = (MULTIPLIER.wrapping_mul(self.x)) % MODULUS; + Some(self.x) + } +} + +fn random_iter() -> RandomIter { + RandomIter { x: 20170705 } +} + +fn create_random(c: &mut Criterion) { const N: u32 = 5_000; // Clamp values so we get some re-use of containers const MAX: u32 = 8 * (u16::MAX as u32 + 1); @@ -221,16 +257,7 @@ fn random_iter(c: &mut Criterion) { let mut group = c.benchmark_group("random_iter"); group.throughput(Throughput::Elements(N.into())); - let rand_iter = { - const MULTIPLIER: u32 = 742938285; - const MODULUS: u32 = (1 << 31) - 1; - // Super simple LCG iterator - let mut z = 20170705; // seed - std::iter::from_fn(move || { - z = (MULTIPLIER * z) % MODULUS; - Some(z % MAX) - }) - }; + let rand_iter = random_iter(); group.bench_function("random_adds", |b| { b.iter(|| { @@ -252,6 +279,96 @@ fn random_iter(c: &mut Criterion) { }); } +fn collect_bitmap64_to_vec(c: &mut Criterion) { + const N: u64 = 1_000_000; + + let mut group = c.benchmark_group("collect_bitmap64_to_vec"); + group.throughput(Throughput::Elements(N.into())); + let bitmap = Bitmap64::from_range(0..N); + group.bench_function("to_vec", |b| { + b.iter_batched(|| (), |()| bitmap.to_vec(), BatchSize::LargeInput); + }); + group.bench_function("foreach", |b| { + b.iter_batched( + || (), + |()| { + let mut vec = Vec::with_capacity(bitmap.cardinality() as usize); + bitmap.for_each(|item| -> ControlFlow<()> { + vec.push(item); + ControlFlow::Continue(()) + }); + vec + }, + BatchSize::LargeInput, + ); + }); + group.bench_function("iter", |b| { + b.iter_batched( + || (), + |()| { + let mut vec = Vec::with_capacity(bitmap.cardinality() as usize); + vec.extend(bitmap.iter()); + vec + }, + BatchSize::LargeInput, + ); + }); + group.bench_function("iter_many", |b| { + b.iter_batched( + || (), + |()| { + let mut vec = vec![0; bitmap.cardinality() as usize]; + let mut iter = bitmap.cursor(); + assert_eq!(iter.read_many(&mut vec), vec.len()); + vec + }, + BatchSize::LargeInput, + ); + }); + + group.finish(); +} + +fn iterate_bitmap64(c: &mut Criterion) { + const N: u64 = 1_000_000; + const END_ITER: u64 = N - 100; + + let mut group = c.benchmark_group("bitmap64_iterate"); + group.throughput(Throughput::Elements(N.into())); + let bitmap = Bitmap64::from_range(0..N); + group.bench_function("iter", |b| { + b.iter(|| { + for x in bitmap.iter() { + if x == END_ITER { + break; + } + } + }) + }); + group.bench_function("cursor", |b| { + b.iter(|| { + let mut cursor = bitmap.cursor(); + while let Some(x) = cursor.next() { + if x == END_ITER { + break; + } + } + }) + }); + group.bench_function("for_each", |b| { + b.iter(|| { + bitmap.for_each(|x| -> ControlFlow<()> { + if x == END_ITER { + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + }) + }) + }); + + group.finish(); +} + criterion_group!( benches, new, @@ -269,6 +386,8 @@ criterion_group!( serialize, deserialize, bulk_new, - random_iter, + create_random, + collect_bitmap64_to_vec, + iterate_bitmap64, ); criterion_main!(benches); diff --git a/croaring/src/bitmap/imp.rs b/croaring/src/bitmap/imp.rs index b946e8e..0a0ffa3 100644 --- a/croaring/src/bitmap/imp.rs +++ b/croaring/src/bitmap/imp.rs @@ -1,9 +1,9 @@ +use crate::callback::CallbackWrapper; use crate::Bitset; use ffi::roaring_bitmap_t; -use std::convert::TryInto; use std::ffi::{c_void, CStr}; -use std::ops::{Bound, RangeBounds}; -use std::{mem, ptr}; +use std::ops::{Bound, ControlFlow, RangeBounds}; +use std::{mem, panic, ptr}; use super::serialization::{Deserializer, Serializer}; use super::{Bitmap, Statistics}; @@ -21,7 +21,7 @@ impl Bitmap { // (it can be moved safely), and can be freed with `free`, without freeing the underlying // containers and auxiliary data. Ensure this is still valid every time we update // the version of croaring. - const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 2 && ffi::ROARING_VERSION_MINOR == 0); + const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 3 && ffi::ROARING_VERSION_MINOR == 0); ffi::roaring_free(p.cast::()); result } @@ -279,6 +279,29 @@ impl Bitmap { unsafe { ffi::roaring_bitmap_remove_checked(&mut self.bitmap, element) } } + /// Remove many values from the bitmap + /// + /// This should be faster than calling `remove` multiple times. + /// + /// In order to exploit this optimization, the caller should attempt to keep values with the same high 48 bits of + /// the value as consecutive elements in `vals` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// bitmap.remove_many(&[1, 2, 3, 4, 5, 6, 7, 8]); + /// assert_eq!(bitmap.to_vec(), vec![9]); + /// ``` + #[inline] + #[doc(alias = "roaring_bitmap_remove_many")] + pub fn remove_many(&mut self, elements: &[u32]) { + unsafe { + ffi::roaring_bitmap_remove_many(&mut self.bitmap, elements.len(), elements.as_ptr()) + } + } + /// Contains returns true if the integer element is contained in the bitmap /// /// # Examples @@ -721,6 +744,49 @@ impl Bitmap { unsafe { ffi::roaring_bitmap_flip_inplace(&mut self.bitmap, start, end) } } + /// Iterate over the values in the bitmap in sorted order + /// + /// If `f` returns `Break`, iteration will stop and the value will be returned, + /// Otherwise, iteration continues. If `f` never returns break, `None` is returned after all values are visited. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// use std::ops::ControlFlow; + /// + /// let bitmap = Bitmap::of(&[1, 2, 3, 14, 20, 21, 100]); + /// let mut even_nums_under_50 = vec![]; + /// + /// let first_over_50 = bitmap.for_each(|value| { + /// if value > 50 { + /// return ControlFlow::Break(value); + /// } + /// if value % 2 == 0 { + /// even_nums_under_50.push(value); + /// } + /// ControlFlow::Continue(()) + /// }); + /// + /// assert_eq!(even_nums_under_50, vec![2, 14, 20]); + /// assert_eq!(first_over_50, ControlFlow::Break(100)); + /// ``` + #[inline] + pub fn for_each(&self, f: F) -> ControlFlow + where + F: FnMut(u32) -> ControlFlow, + { + let mut callback_wrapper = CallbackWrapper::new(f); + let (callback, context) = callback_wrapper.callback_and_ctx(); + unsafe { + ffi::roaring_iterate(&self.bitmap, Some(callback), context); + } + match callback_wrapper.result() { + Ok(cf) => cf, + Err(e) => panic::resume_unwind(e), + } + } + /// Returns a vector containing all of the integers stored in the Bitmap /// in sorted order. /// @@ -922,6 +988,13 @@ impl Bitmap { /// // Exclusive ranges still step from the start, but do not include it /// let bitmap = Bitmap::from_range_with_step((Bound::Excluded(10), Bound::Included(30)), 10); /// assert_eq!(bitmap.to_vec(), [20, 30]); + /// + /// // Ranges including max value + /// let bitmap = Bitmap::from_range_with_step((u32::MAX - 1)..=u32::MAX, 1); + /// assert_eq!(bitmap.to_vec(), vec![u32::MAX - 1, u32::MAX]); + /// + /// let bitmap = Bitmap::from_range_with_step((u32::MAX - 1)..=u32::MAX, 3); + /// assert_eq!(bitmap.to_vec(), vec![u32::MAX - 1]); /// ``` #[inline] #[doc(alias = "roaring_bitmap_from_range")] diff --git a/croaring/src/bitmap/iter.rs b/croaring/src/bitmap/iter.rs index 0530e81..7925107 100644 --- a/croaring/src/bitmap/iter.rs +++ b/croaring/src/bitmap/iter.rs @@ -1,48 +1,355 @@ -use std::iter::{FromIterator, IntoIterator}; use std::marker::PhantomData; use std::mem::MaybeUninit; use super::Bitmap; -/// Iterator over the values of a bitmap -#[derive(Clone)] -pub struct BitmapIterator<'a> { - iterator: ffi::roaring_uint32_iterator_s, - phantom: PhantomData<&'a Bitmap>, +/// A cusrsr over the values of a bitmap +/// +/// A Cursor is like an iterator, except that it can freely seek back-and-forth. +/// +/// A cursor points at a single value in the bitmap, or at a "ghost" position, +/// either one before the beginning of the bitmap, or one after the end of the bitmap. +#[derive(Debug, Clone)] +pub struct BitmapCursor<'a> { + raw: ffi::roaring_uint32_iterator_t, + _bitmap: PhantomData<&'a Bitmap>, } -unsafe impl Send for BitmapIterator<'_> {} -unsafe impl Sync for BitmapIterator<'_> {} +unsafe impl Send for BitmapCursor<'_> {} -impl<'a> BitmapIterator<'a> { - fn new(bitmap: &'a Bitmap) -> Self { - let mut iterator = MaybeUninit::::uninit(); - unsafe { - ffi::roaring_init_iterator(&bitmap.bitmap, iterator.as_mut_ptr()); - } - BitmapIterator { - iterator: unsafe { iterator.assume_init() }, - phantom: PhantomData, +unsafe impl Sync for BitmapCursor<'_> {} + +impl<'a> BitmapCursor<'a> { + #[inline] + fn from_raw(raw: ffi::roaring_uint32_iterator_t) -> Self { + BitmapCursor { + raw, + _bitmap: PhantomData, } } + fn at_first(bitmap: &'a Bitmap) -> Self { + let mut raw = MaybeUninit::::uninit(); + unsafe { ffi::roaring_iterator_init(&bitmap.bitmap, raw.as_mut_ptr()) }; + Self::from_raw(unsafe { raw.assume_init() }) + } + + fn at_last(bitmap: &'a Bitmap) -> Self { + let mut raw = MaybeUninit::::uninit(); + unsafe { ffi::roaring_iterator_init_last(&bitmap.bitmap, raw.as_mut_ptr()) }; + Self::from_raw(unsafe { raw.assume_init() }) + } + + /// Returns true if the cursor is pointing at a value in the bitmap. + /// + /// If this returns false, then the cursor is pointing at a "ghost" position, + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::new(); + /// assert!(!bitmap.cursor().has_value()); + /// + /// bitmap.add(1); + /// let mut cursor = bitmap.cursor(); + /// assert!(cursor.has_value()); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert!(!cursor.has_value()); + /// ``` #[inline] - fn current_value(&self) -> Option { + pub fn has_value(&self) -> bool { + self.raw.has_value + } + + /// Returns the value at the cursor, if any. + /// + /// If the cursor is not pointing at a value, then this returns `None`. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::new(); + /// bitmap.add(1); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn current(&self) -> Option { if self.has_value() { - Some(self.iterator.current_value) + Some(self.raw.current_value) } else { None } } + /// Moves the cursor to the next value in the bitmap + /// + /// If the cursor is already past the end of the bitmap, then this does nothing. + /// + /// If the cursor is at the ghost position before the beginning of the bitmap, + /// then this moves the cursor to the first value in the bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), None); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(2)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn move_next(&mut self) { + unsafe { ffi::roaring_uint32_iterator_advance(&mut self.raw) }; + } + + /// Moves the cursor to the next value in the bitmap, and returns the value (if any) + /// + /// This is equivalent to calling [`Self::move_next`] followed by [`Self::current`]. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// assert_eq!(cursor.next(), Some(2)); + /// assert_eq!(cursor.next(), Some(3)); + /// assert_eq!(cursor.next(), None); + /// ``` + #[inline] + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option { + self.move_next(); + self.current() + } + + /// Moves the cursor to the previous value in the bitmap + /// + /// If the cursor is already before the beginning of the bitmap, then this does nothing. + /// + /// If the cursor is at the ghost position after the end of the bitmap, + /// then this moves the cursor to the last value in the bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(2)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn move_prev(&mut self) { + unsafe { ffi::roaring_uint32_iterator_previous(&mut self.raw) }; + } + + /// Moves the cursor to the previous value in the bitmap, and returns the value (if any) + /// + /// This is equivalent to calling [`Self::move_prev`] followed by [`Self::current`]. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// assert_eq!(cursor.prev(), Some(2)); + /// assert_eq!(cursor.prev(), Some(1)); + /// assert_eq!(cursor.prev(), None); + /// ``` + #[inline] + pub fn prev(&mut self) -> Option { + self.move_prev(); + self.current() + } + + /// Resets this cursor to the first value in the bitmap. + /// + /// The bitmap does not have to be the same bitmap that this cursor was created from: + /// this allows you to reuse a cursor for multiple bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap1 = Bitmap::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap::of(&[4, 5, 6]); + /// let cursor = bitmap1.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// let cursor = cursor.reset_to_first(&bitmap2); + /// assert_eq!(cursor.current(), Some(4)); + /// // Cursor is no longer borrowing from bitmap1 + /// bitmap1.add(100); + /// ``` + #[must_use] + pub fn reset_to_first(self, bitmap: &Bitmap) -> BitmapCursor<'_> { + BitmapCursor::at_first(bitmap) + } + + /// Resets this cursor to the last value in the bitmap. + /// + /// The bitmap does not have to be the same bitmap that this cursor was created from: + /// this allows you to reuse a cursor for multiple bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap1 = Bitmap::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap::of(&[4, 5, 6]); + /// let cursor = bitmap1.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// let cursor = cursor.reset_to_last(&bitmap2); + /// assert_eq!(cursor.current(), Some(6)); + /// ``` + #[must_use] + pub fn reset_to_last(self, bitmap: &Bitmap) -> BitmapCursor<'_> { + BitmapCursor::at_last(bitmap) + } + + /// Attempt to read many values from the iterator into `dst` + /// + /// The current value _is_ included in the output. + /// + /// Returns the number of items read from the iterator, may be `< dst.len()` iff + /// the iterator is exhausted or `dst.len() > u32::MAX`. + /// + /// This can be much more efficient than repeated iteration. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// + /// let mut bitmap = Bitmap::new(); + /// bitmap.add_range(0..100); + /// bitmap.add(222); + /// bitmap.add(555); + /// bitmap.add(999); + /// + /// let mut buf = [0; 100]; + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.read_many(&mut buf), 100); + /// // Get the first 100 items, from the original range added + /// for (i, item) in buf.iter().enumerate() { + /// assert_eq!(*item, i as u32); + /// } + /// // Calls to next_many() can be interleaved with other cursor calls + /// assert_eq!(cursor.current(), Some(222)); + /// assert_eq!(cursor.next(), Some(555)); + /// assert_eq!(cursor.read_many(&mut buf), 2); + /// assert_eq!(buf[0], 555); + /// assert_eq!(buf[1], 999); + /// + /// assert_eq!(cursor.current(), None); + /// assert_eq!(cursor.read_many(&mut buf), 0); + /// ``` + /// + /// ``` + /// use croaring::Bitmap; + /// + /// fn print_by_chunks(bitmap: &Bitmap) { + /// let mut buf = [0; 1024]; + /// let mut iter = bitmap.cursor(); + /// loop { + /// let n = iter.read_many(&mut buf); + /// if n == 0 { + /// break; + /// } + /// println!("{:?}", &buf[..n]); + /// } + /// } + /// + /// # print_by_chunks(&Bitmap::of(&[1, 2, 8, 20, 1000])); + /// ``` + #[inline] + #[doc(alias = "roaring_uint32_iterator_read")] + pub fn read_many(&mut self, dst: &mut [u32]) -> usize { + let count = u32::try_from(dst.len()).unwrap_or(u32::MAX); + let result = + unsafe { ffi::roaring_uint32_iterator_read(&mut self.raw, dst.as_mut_ptr(), count) }; + debug_assert!(result <= count); + result as usize + } + + /// Reset the iterator to the first value `>= val` + /// + /// This can move the iterator forwards or backwards. + /// + /// # Examples + /// ``` + /// use croaring::Bitmap; + /// + /// let bitmap = Bitmap::of(&[0, 1, 100, 1000, u32::MAX]); + /// let mut cursor = bitmap.cursor(); + /// cursor.reset_at_or_after(0); + /// assert_eq!(cursor.current(), Some(0)); + /// cursor.reset_at_or_after(0); + /// assert_eq!(cursor.current(), Some(0)); + /// + /// cursor.reset_at_or_after(101); + /// assert_eq!(cursor.current(), Some(1000)); + /// assert_eq!(cursor.next(), Some(u32::MAX)); + /// assert_eq!(cursor.next(), None); + /// cursor.reset_at_or_after(u32::MAX); + /// assert_eq!(cursor.current(), Some(u32::MAX)); + /// assert_eq!(cursor.next(), None); + /// ``` + #[inline] + #[doc(alias = "roaring_uint32_iterator_move_equalorlarger")] + pub fn reset_at_or_after(&mut self, val: u32) { + unsafe { ffi::roaring_uint32_iterator_move_equalorlarger(&mut self.raw, val) }; + } +} + +/// Iterator over the values of a bitmap +#[derive(Clone)] +pub struct BitmapIterator<'a> { + cursor: BitmapCursor<'a>, +} + +impl<'a> BitmapIterator<'a> { + fn new(bitmap: &'a Bitmap) -> Self { + Self { + cursor: BitmapCursor::at_first(bitmap), + } + } + #[inline] - fn has_value(&self) -> bool { - self.iterator.has_value + fn current_value(&self) -> Option { + self.cursor.current() } #[inline] - fn advance(&mut self) -> bool { - unsafe { ffi::roaring_advance_uint32_iterator(&mut self.iterator) } + fn advance(&mut self) { + self.cursor.move_next(); } /// Attempt to read many values from the iterator into `dst` @@ -96,14 +403,9 @@ impl<'a> BitmapIterator<'a> { /// # print_by_chunks(&Bitmap::of(&[1, 2, 8, 20, 1000])); /// ``` #[inline] - #[doc(alias = "roaring_read_uint32_iterator")] + #[doc(alias = "roaring_uint32_iterator_read")] pub fn next_many(&mut self, dst: &mut [u32]) -> usize { - let count: u32 = u32::try_from(dst.len()).unwrap_or(u32::MAX); - let result = unsafe { - ffi::roaring_read_uint32_iterator(&mut self.iterator, dst.as_mut_ptr(), count) - }; - debug_assert!(result <= count); - result as usize + self.cursor.read_many(dst) } /// Reset the iterator to the first value `>= val` @@ -130,9 +432,25 @@ impl<'a> BitmapIterator<'a> { /// assert_eq!(iter.next(), None); /// ``` #[inline] - #[doc(alias = "roaring_move_uint32_iterator_equalorlarger")] + #[doc(alias = "roaring_uint32_iterator_move_equalorlarger")] pub fn reset_at_or_after(&mut self, val: u32) { - unsafe { ffi::roaring_move_uint32_iterator_equalorlarger(&mut self.iterator, val) }; + self.cursor.reset_at_or_after(val); + } + + /// Peek at the next value to be returned by the iterator (if any), without consuming it + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap; + /// let mut bitmap = Bitmap::of(&[1, 2, 3]); + /// let mut iter = bitmap.iter(); + /// assert_eq!(iter.peek(), Some(1)); + /// assert_eq!(iter.next(), Some(1)); + /// ``` + #[inline] + pub fn peek(&self) -> Option { + self.cursor.current() } } @@ -177,6 +495,70 @@ impl Bitmap { pub fn iter(&self) -> BitmapIterator { BitmapIterator::new(self) } + + /// Returns a cursor pointing at the first value in the bitmap. + /// + /// See [`BitmapCursor`] for more details. + #[inline] + #[must_use] + pub fn cursor(&self) -> BitmapCursor { + BitmapCursor::at_first(self) + } + + /// Returns a cursor pointing at the last value in the bitmap. + /// + /// See [`BitmapCursor`] for more details. + #[inline] + #[must_use] + pub fn cursor_to_last(&self) -> BitmapCursor { + BitmapCursor::at_last(self) + } +} + +/// Converts this iterator into a cursor +/// +/// The cursor's current value will be the the item which would have been returned by the next call to `next()` +/// or one past the end of the bitmap if the iterator is exhausted. +/// +/// # Examples +/// +/// ``` +/// use croaring::bitmap::{Bitmap, BitmapCursor}; +/// let mut bitmap = Bitmap::of(&[1, 2, 3]); +/// let mut iter = bitmap.iter(); +/// assert_eq!(iter.peek(), Some(1)); +/// assert_eq!(iter.next(), Some(1)); +/// +/// assert_eq!(iter.peek(), Some(2)); +/// let mut cursor: BitmapCursor = iter.into(); +/// assert_eq!(cursor.current(), Some(2)); +/// ``` +impl<'a> From> for BitmapCursor<'a> { + fn from(iterator: BitmapIterator<'a>) -> Self { + iterator.cursor + } +} + +/// Converts this cursor into an iterator +/// +/// The next value returned by the iterator will be the current value of the cursor (if any). +/// +/// # Examples +/// +/// ``` +/// use croaring::bitmap::{Bitmap, BitmapIterator}; +/// +/// let mut bitmap = Bitmap::of(&[1, 2, 3]); +/// let mut cursor = bitmap.cursor(); +/// assert_eq!(cursor.current(), Some(1)); +/// +/// let mut iter = BitmapIterator::from(cursor); +/// assert_eq!(iter.next(), Some(1)); +/// ``` +impl<'a> From> for BitmapIterator<'a> { + fn from(cursor: BitmapCursor<'a>) -> Self { + BitmapIterator { cursor } + } } impl FromIterator for Bitmap { diff --git a/croaring/src/bitmap/mod.rs b/croaring/src/bitmap/mod.rs index 658fcea..c9239a3 100644 --- a/croaring/src/bitmap/mod.rs +++ b/croaring/src/bitmap/mod.rs @@ -89,6 +89,6 @@ mod ops; mod serialization; mod view; -pub use self::iter::BitmapIterator; +pub use self::iter::{BitmapCursor, BitmapIterator}; pub use self::lazy::LazyBitmap; pub use self::serialization::{Deserializer, Serializer, ViewDeserializer}; diff --git a/croaring/src/bitmap/ops.rs b/croaring/src/bitmap/ops.rs index b499dfe..c7c628d 100644 --- a/croaring/src/bitmap/ops.rs +++ b/croaring/src/bitmap/ops.rs @@ -126,7 +126,7 @@ impl Drop for Bitmap { fn drop(&mut self) { // This depends somewhat heavily on the implementation of croaring, // Ensure this is still valid every time we update the version of croaring. - const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 2 && ffi::ROARING_VERSION_MINOR == 0); + const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 3 && ffi::ROARING_VERSION_MINOR == 0); // Per https://github.com/RoaringBitmap/CRoaring/blob/4f8dbdb0cc884626b20ef0cc9e891f701fe157cf/cpp/roaring.hh#L182 // > By contract, calling roaring_bitmap_clear() is enough to diff --git a/croaring/src/bitmap/serialization.rs b/croaring/src/bitmap/serialization.rs index 7e19d6f..96e0006 100644 --- a/croaring/src/bitmap/serialization.rs +++ b/croaring/src/bitmap/serialization.rs @@ -42,9 +42,9 @@ pub trait Deserializer { /// /// # Safety /// - /// Unlike its safe counterpart, [`try_deserialize`], this function assumes the data is valid, - /// passing data which does not contain/start with a bitmap serialized with this format will - /// result in undefined behavior. + /// Unlike its safe counterpart ([`Self::try_deserialize`]) this function assumes the data is + /// valid, passing data which does not contain/start with a bitmap serialized with this format + /// will result in undefined behavior. unsafe fn try_deserialize_unchecked(buffer: &[u8]) -> Bitmap; } diff --git a/croaring/src/bitmap/view.rs b/croaring/src/bitmap/view.rs index 543e45d..175cd02 100644 --- a/croaring/src/bitmap/view.rs +++ b/croaring/src/bitmap/view.rs @@ -27,7 +27,7 @@ impl<'a> BitmapView<'a> { // `containers` array is stored immediately after the roaring_bitmap_t data. // Ensure this is still valid every time we update // the version of croaring. - const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 2 && ffi::ROARING_VERSION_MINOR == 0); + const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 3 && ffi::ROARING_VERSION_MINOR == 0); assert!(!p.is_null()); diff --git a/croaring/src/bitmap64/imp.rs b/croaring/src/bitmap64/imp.rs new file mode 100644 index 0000000..9a7e2ee --- /dev/null +++ b/croaring/src/bitmap64/imp.rs @@ -0,0 +1,1316 @@ +use crate::bitmap64::Bitmap64; +use crate::bitmap64::{Deserializer, Serializer}; +use crate::callback::CallbackWrapper; +use std::collections::Bound; +use std::ffi::CStr; +use std::mem::MaybeUninit; +use std::ops::{ControlFlow, RangeBounds}; +use std::panic; +use std::ptr; +use std::ptr::NonNull; + +impl Bitmap64 { + #[inline] + pub(crate) unsafe fn take_heap(p: *mut ffi::roaring64_bitmap_t) -> Self { + let raw = NonNull::new(p).expect("non-null ptr"); + Self { raw } + } + + /// Create a new empty bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap = Bitmap64::new(); + /// assert_eq!(bitmap.cardinality(), 0); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_create")] + pub fn new() -> Self { + unsafe { Self::take_heap(ffi::roaring64_bitmap_create()) } + } + + /// Creates a new bitmap from a slice of u64 integers + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap = Bitmap64::of(&[1, 2, 3]); + /// assert_eq!(bitmap.cardinality(), 3); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_of_ptr")] + pub fn of(slice: &[u64]) -> Self { + unsafe { Self::take_heap(ffi::roaring64_bitmap_of_ptr(slice.len(), slice.as_ptr())) } + } + + /// Create a new bitmap containing all the values in a range + #[inline] + #[doc(alias = "roaring64_bitmap_from_range")] + pub fn from_range>(range: R) -> Self { + Self::from_range_with_step(range, 1) + } + + /// Create a new bitmap containing all the values in `range` which are a multiple of `step` away from the lower + /// bound + /// + /// If `step` is 0 or there are no values which are a multiple of `step` away from the lower bound within range, + /// an empty bitmap is returned + /// + /// # Examples + /// + /// ``` + /// use std::ops::Bound; + /// use croaring::Bitmap64; + /// let bitmap = Bitmap64::from_range_with_step(0..10, 3); + /// assert_eq!(bitmap.to_vec(), vec![0, 3, 6, 9]); + /// + /// // empty ranges + /// assert_eq!(Bitmap64::from_range_with_step(0..0, 1), Bitmap64::new()); + /// assert_eq!(Bitmap64::from_range_with_step(100..=0, 0), Bitmap64::new()); + /// + /// // step is 0 + /// assert_eq!(Bitmap64::from_range_with_step(0..10, 0), Bitmap64::new()); + /// + /// // No values of step in range + /// let bitmap = Bitmap64::from_range_with_step((Bound::Excluded(0), Bound::Included(10)), 100); + /// assert_eq!(bitmap, Bitmap64::new()); + /// let bitmap = Bitmap64::from_range_with_step((Bound::Excluded(u64::MAX), Bound::Included(u64::MAX)), 1); + /// assert_eq!(bitmap, Bitmap64::new()); + /// + /// // Exclusive ranges still step from the start, but do not include it + /// let bitmap = Bitmap64::from_range_with_step((Bound::Excluded(10), Bound::Included(30)), 10); + /// assert_eq!(bitmap.to_vec(), vec![20, 30]); + /// + /// // Ranges including max value + /// let bitmap = Bitmap64::from_range_with_step((u64::MAX - 1)..=u64::MAX, 1); + /// assert_eq!(bitmap.to_vec(), vec![u64::MAX - 1, u64::MAX]); + /// let bitmap = Bitmap64::from_range_with_step((u64::MAX - 1)..=u64::MAX, 3); + /// assert_eq!(bitmap.to_vec(), vec![u64::MAX - 1]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_from_range")] + pub fn from_range_with_step>(range: R, step: u64) -> Self { + // This can't use `range_to_exclusive` because when the start is excluded, we want + // to start at the next step, not one more + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => match i.checked_add(step) { + Some(i) => i, + None => return Self::new(), + }, + Bound::Unbounded => 0, + }; + let end_inclusive = match range.end_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => match i.checked_sub(1) { + Some(i) => i, + None => return Self::new(), + }, + Bound::Unbounded => u64::MAX, + }; + // roaring64_bitmap_from_range takes an exclusive range, + // so we need to handle the case where the range should include u64::MAX, + // and manually add it in afterwards since there's no way to set it with an exclusive range + let (end, add_max) = match end_inclusive.checked_add(1) { + Some(i) => (i, false), + None => (u64::MAX, (u64::MAX - start) % step == 0), + }; + + unsafe { + let result = ffi::roaring64_bitmap_from_range(start, end, step); + if result.is_null() { + Self::new() + } else { + let mut result = Self::take_heap(result); + if add_max { + result.add(u64::MAX); + } + result + } + } + } + + /// Add a value to the bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add(1); + /// assert!(bitmap.contains(1)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_add")] + pub fn add(&mut self, value: u64) { + unsafe { ffi::roaring64_bitmap_add(self.raw.as_ptr(), value) } + } + + /// Add the integer element to the bitmap. Returns true if the value was + /// added, false if the value was already in the bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::new(); + /// assert!(bitmap.add_checked(1)); + /// assert!(!bitmap.add_checked(1)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_add_checked")] + pub fn add_checked(&mut self, value: u64) -> bool { + unsafe { ffi::roaring64_bitmap_add_checked(self.raw.as_ptr(), value) } + } + + /// Add many values to the bitmap + /// + /// See also [`Bitmap64::extend`] + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add_many(&[1, 2, 3]); + /// + /// assert!(bitmap.contains(1)); + /// assert!(bitmap.contains(2)); + /// assert!(bitmap.contains(3)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_add_many")] + pub fn add_many(&mut self, values: &[u64]) { + unsafe { ffi::roaring64_bitmap_add_many(self.raw.as_ptr(), values.len(), values.as_ptr()) } + } + + /// Add all values in range + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add_range(1..3); + /// + /// assert!(bitmap.contains(1)); + /// assert!(bitmap.contains(2)); + /// assert!(!bitmap.contains(3)); + /// + /// let mut bitmap2 = Bitmap64::new(); + /// bitmap2.add_range(3..1); + /// assert!(bitmap2.is_empty()); + /// + /// let mut bitmap3 = Bitmap64::new(); + /// bitmap3.add_range(3..3); + /// assert!(bitmap3.is_empty()); + /// + /// let mut bitmap4 = Bitmap64::new(); + /// bitmap4.add_range(..=2); + /// bitmap4.add_range(u64::MAX..=u64::MAX); + /// assert!(bitmap4.contains(0)); + /// assert!(bitmap4.contains(1)); + /// assert!(bitmap4.contains(2)); + /// assert!(bitmap4.contains(u64::MAX)); + /// assert_eq!(bitmap4.cardinality(), 4); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_add_range_closed")] + pub fn add_range>(&mut self, range: R) { + let (start, end) = range_to_inclusive(range); + unsafe { ffi::roaring64_bitmap_add_range_closed(self.raw.as_ptr(), start, end) } + } + + /// Remove a value from the bitmap if present + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// bitmap.remove(2); + /// assert!(!bitmap.contains(2)); + /// bitmap.remove(99); // It is not an error to remove a value not in the bitmap + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_remove")] + pub fn remove(&mut self, value: u64) { + unsafe { ffi::roaring64_bitmap_remove(self.raw.as_ptr(), value) } + } + + /// Remove a value from the bit map if present, and return if the value was previously present + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// assert!(bitmap.remove_checked(2)); + /// assert!(!bitmap.remove_checked(2)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_remove_checked")] + pub fn remove_checked(&mut self, value: u64) -> bool { + unsafe { ffi::roaring64_bitmap_remove_checked(self.raw.as_ptr(), value) } + } + + /// Remove many values from the bitmap + /// + /// This should be faster than calling `remove` multiple times. + /// + /// In order to exploit this optimization, the caller should attempt to keep values with the same high 48 bits of + /// the value as consecutive elements in `vals` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// bitmap.remove_many(&[1, 2, 3, 4, 5, 6, 7, 8]); + /// assert_eq!(bitmap.to_vec(), vec![9]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_remove_many")] + pub fn remove_many(&mut self, vals: &[u64]) { + unsafe { ffi::roaring64_bitmap_remove_many(self.raw.as_ptr(), vals.len(), vals.as_ptr()) } + } + + /// Remove all values from the specified iterator + /// + /// This should be faster than calling `remove` multiple times. + /// + /// In order to exploit this optimization, the caller should attempt to keep values with the same high 48 bits of + /// the value as consecutive elements in `it` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3, 4, 5, 6, 7, 8, 9]); + /// bitmap.remove_all(1..=8); // Remove all values from iterator + /// assert_eq!(bitmap.to_vec(), vec![9]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_remove_bulk")] + pub fn remove_all(&mut self, it: It) + where + It: IntoIterator, + { + let mut ctx = MaybeUninit::::zeroed(); + it.into_iter().for_each(|value| unsafe { + ffi::roaring64_bitmap_remove_bulk(self.raw.as_ptr(), ctx.as_mut_ptr(), value); + }); + } + + /// Remove all values in range + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add_range(1..4); + /// assert_eq!(bitmap.to_vec(), vec![1, 2, 3]); + /// + /// bitmap.remove_range(1..=2); + /// assert_eq!(bitmap.to_vec(), vec![3]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_remove_range_closed")] + pub fn remove_range>(&mut self, range: R) { + let (start, end) = range_to_inclusive(range); + unsafe { ffi::roaring64_bitmap_remove_range_closed(self.raw.as_ptr(), start, end) } + } + + /// Empty the bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::from([1, 2, 3]); + /// assert!(!bitmap.is_empty()); + /// bitmap.clear(); + /// assert!(bitmap.is_empty()); + /// ``` + #[inline] + pub fn clear(&mut self) { + self.remove_range(..); + } + + /// Returns the number of values in the bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// assert_eq!(bitmap.cardinality(), 0); + /// bitmap.add(1); + /// assert_eq!(bitmap.cardinality(), 1); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_get_cardinality")] + pub fn cardinality(&self) -> u64 { + unsafe { ffi::roaring64_bitmap_get_cardinality(self.raw.as_ptr()) } + } + + /// Returns the number of values in the bitmap in the given `range` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 3, 4, u64::MAX]); + /// + /// assert_eq!(bitmap.range_cardinality(..1), 0); + /// assert_eq!(bitmap.range_cardinality(..2), 1); + /// assert_eq!(bitmap.range_cardinality(2..5), 2); + /// assert_eq!(bitmap.range_cardinality(..5), 3); + /// assert_eq!(bitmap.range_cardinality(1..=4), 3); + /// + /// assert_eq!(bitmap.range_cardinality(4..=u64::MAX), 2); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_range_cardinality")] + pub fn range_cardinality>(&self, range: R) -> u64 { + let Some(exclusive_range) = range_to_exclusive(range) else { + return 0; + }; + self._range_cardinality(exclusive_range) + } + + #[inline] + fn _range_cardinality(&self, exclusive_range: ExclusiveRangeRes) -> u64 { + let ExclusiveRangeRes { + start, + end, + needs_max, + } = exclusive_range; + let mut cardinality = + unsafe { ffi::roaring64_bitmap_range_cardinality(self.raw.as_ptr(), start, end) }; + if needs_max { + cardinality += u64::from(self.contains(u64::MAX)); + } + cardinality + } + + /// Returns true if the bitmap is empty + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// assert!(bitmap.is_empty()); + /// bitmap.add(1); + /// assert!(!bitmap.is_empty()); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_is_empty")] + pub fn is_empty(&self) -> bool { + unsafe { ffi::roaring64_bitmap_is_empty(self.raw.as_ptr()) } + } + + /// Returns true if all the elements of self are in other + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1: Bitmap64 = (5..10).collect(); + /// let bitmap2: Bitmap64 = (5..8).collect(); + /// let bitmap3: Bitmap64 = (5..10).collect(); + /// let bitmap4: Bitmap64 = (9..11).collect(); + /// + /// assert!(bitmap2.is_subset(&bitmap1)); + /// assert!(bitmap3.is_subset(&bitmap1)); + /// assert!(!bitmap4.is_subset(&bitmap1)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_is_subset")] + pub fn is_subset(&self, other: &Self) -> bool { + unsafe { ffi::roaring64_bitmap_is_subset(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Returns true if all the elements of self are in other and self is not equal to other + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1: Bitmap64 = (5..9).collect(); + /// let bitmap2: Bitmap64 = (5..8).collect(); + /// let bitmap3: Bitmap64 = (5..10).collect(); + /// let bitmap4: Bitmap64 = (9..11).collect(); + /// + /// assert!(bitmap2.is_strict_subset(&bitmap1)); + /// assert!(!bitmap3.is_strict_subset(&bitmap1)); + /// assert!(!bitmap4.is_strict_subset(&bitmap1)); + /// assert!(!bitmap1.is_strict_subset(&bitmap1)); + /// + pub fn is_strict_subset(&self, other: &Self) -> bool { + unsafe { ffi::roaring64_bitmap_is_strict_subset(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Returns the smallest value in the bitmap, or None if the bitmap is empty + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap: Bitmap64 = (5..10).collect(); + /// let empty_bitmap: Bitmap64 = Bitmap64::new(); + /// + /// assert_eq!(bitmap.minimum(), Some(5)); + /// assert_eq!(empty_bitmap.minimum(), None); + /// + /// bitmap.add(3); + /// + /// assert_eq!(bitmap.minimum(), Some(3)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_minimum")] + pub fn minimum(&self) -> Option { + if self.is_empty() { + None + } else { + Some(unsafe { ffi::roaring64_bitmap_minimum(self.raw.as_ptr()) }) + } + } + + /// Returns the largest value in the bitmap, or None if the bitmap is empty + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap: Bitmap64 = (5..10).collect(); + /// let empty_bitmap: Bitmap64 = Bitmap64::new(); + /// + /// assert_eq!(bitmap.maximum(), Some(9)); + /// assert_eq!(empty_bitmap.maximum(), None); + /// + /// bitmap.add(15); + /// + /// assert_eq!(bitmap.maximum(), Some(15)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_maximum")] + pub fn maximum(&self) -> Option { + if self.is_empty() { + None + } else { + Some(unsafe { ffi::roaring64_bitmap_maximum(self.raw.as_ptr()) }) + } + } + + /// Attempt to compress the bitmap by finding runs of consecutive values + /// + /// Returns true if the bitmap has at least one run container after optimization + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap: Bitmap64 = (100..1000).collect(); + /// assert_eq!(bitmap.cardinality(), 900); + /// assert!(bitmap.run_optimize()); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_run_optimize")] + pub fn run_optimize(&mut self) -> bool { + unsafe { ffi::roaring64_bitmap_run_optimize(self.raw.as_ptr()) } + } + + /// Returns true if the element is contained in the bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// assert!(!bitmap.contains(1)); + /// bitmap.add(1); + /// assert!(bitmap.contains(1)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_contains")] + pub fn contains(&self, value: u64) -> bool { + unsafe { ffi::roaring64_bitmap_contains(self.raw.as_ptr(), value) } + } + + /// Check whether a range of values of range are ALL present + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap = Bitmap64::of(&[1, 2, 4]); + /// assert!(bitmap.contains_range(1..=2)); + /// assert!(!bitmap.contains_range(1..=4)); + /// + /// let mut bitmap = bitmap.clone(); + /// bitmap.add(u64::MAX - 1); + /// bitmap.add(u64::MAX); + /// assert!(bitmap.contains_range((u64::MAX - 1)..=u64::MAX)); + /// + /// // Empty ranges are always contained + /// assert!(bitmap.contains_range(10..0)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_contains_range")] + pub fn contains_range>(&self, range: R) -> bool { + let Some(exclusive_range) = range_to_exclusive(range) else { + return true; + }; + self._contains_range(exclusive_range) + } + + #[inline] + fn _contains_range(&self, exclusive_range: ExclusiveRangeRes) -> bool { + let ExclusiveRangeRes { + start, + end, + needs_max, + } = exclusive_range; + + if needs_max && !self.contains(u64::MAX) { + return false; + } + unsafe { ffi::roaring64_bitmap_contains_range(self.raw.as_ptr(), start, end) } + } + + /// Selects the element at index 'rank' where the smallest element is at index 0 + /// + /// If the size of the bitmap is strictly greater than rank, then this function returns the element of the given + /// rank, otherwise, it returns None + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap: Bitmap64 = (5..10).collect(); + /// + /// assert_eq!(bitmap.select(0), Some(5)); + /// assert_eq!(bitmap.select(1), Some(6)); + /// assert_eq!(bitmap.select(2), Some(7)); + /// assert_eq!(bitmap.select(3), Some(8)); + /// assert_eq!(bitmap.select(4), Some(9)); + /// assert_eq!(bitmap.select(5), None); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_select")] + pub fn select(&self, rank: u64) -> Option { + let mut element = 0u64; + let has_elem: bool = + unsafe { ffi::roaring64_bitmap_select(self.raw.as_ptr(), rank, &mut element) }; + + has_elem.then_some(element) + } + + /// Returns the number of integers that are smaller or equal to x + /// + /// If x is the first element, this function will return 1. If x is smaller than the smallest element, this + /// function will return 0 + /// + /// The indexing convention differs between [`Self::select`] and [`Self::rank`]: [`Self::select`] refers to the + /// smallest value as having index 0, whereas [`Self::rank`] returns 1 when ranking the smallest value + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap: Bitmap64 = (5..10).collect(); + /// + /// assert_eq!(bitmap.rank(8), 4); + /// + /// assert_eq!(bitmap.rank(11), 5); + /// assert_eq!(bitmap.rank(15), 5); + /// + /// bitmap.add(15); + /// + /// assert_eq!(bitmap.rank(11), 5); + /// assert_eq!(bitmap.rank(15), 6); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_rank")] + pub fn rank(&self, value: u64) -> u64 { + unsafe { ffi::roaring64_bitmap_rank(self.raw.as_ptr(), value) } + } + + /// Returns the index of x in the given roaring bitmap. + /// + /// If the roaring bitmap doesn't contain x, this function will return None. + /// The difference with the [rank][Self::rank] function is that this function + /// will return None when x is not the element of roaring bitmap, but the rank + /// function will return the the number of items less than x, and would require + /// a call to [contains][Self::contains] to check if x is in the roaring bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::from_range(5..10); + /// assert_eq!(bitmap.position(4), None); + /// assert_eq!(bitmap.position(5), Some(0)); + /// assert_eq!(bitmap.position(9), Some(4)); + /// assert_eq!(bitmap.position(10), None); + /// assert_eq!(bitmap.position(9999), None); + /// + /// // rank returns the number of values smaller or equal to x, so it always returns a value, and + /// // returns `position + 1` when x is contained in the bitmap. + /// assert_eq!(bitmap.rank(4), 0); + /// assert_eq!(bitmap.rank(5), 1); + /// assert_eq!(bitmap.rank(9), 5); + /// assert_eq!(bitmap.rank(10), 5); + /// assert_eq!(bitmap.rank(9999), 5); + /// + /// let pos = bitmap.position(7).unwrap(); + /// assert_eq!(bitmap.select(pos), Some(7)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_get_index")] + #[doc(alias = "index")] + pub fn position(&self, value: u64) -> Option { + let mut index = 0u64; + let has_index: bool = + unsafe { ffi::roaring64_bitmap_get_index(self.raw.as_ptr(), value, &mut index) }; + + has_index.then_some(index) + } + + /// Negates the bits in the given range + /// any integer present in this range and in the bitmap is removed. + /// Returns result as a new bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1 = Bitmap64::of(&[4]); + /// + /// let bitmap2 = bitmap1.flip(1..3); + /// + /// assert_eq!(bitmap2.cardinality(), 3); + /// assert!(bitmap2.contains(1)); + /// assert!(bitmap2.contains(2)); + /// assert!(!bitmap2.contains(3)); + /// assert!(bitmap2.contains(4)); + /// + /// let bitmap3 = bitmap1.flip(1..=5); + /// assert_eq!(bitmap3.to_vec(), [1, 2, 3, 5]) + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_flip")] + #[doc(alias = "roaring64_bitmap_flip_closed")] + #[must_use] + pub fn flip>(&self, range: R) -> Self { + let (start, end) = range_to_inclusive(range); + unsafe { + Self::take_heap(ffi::roaring64_bitmap_flip_closed( + self.raw.as_ptr(), + start, + end, + )) + } + } + + /// Negates the bits in the given range + /// any integer present in this range and in the bitmap is removed. + /// Stores the result in the current bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::of(&[4]); + /// bitmap1.flip_inplace(1..3); + /// + /// assert_eq!(bitmap1.cardinality(), 3); + /// assert!(bitmap1.contains(1)); + /// assert!(bitmap1.contains(2)); + /// assert!(!bitmap1.contains(3)); + /// assert!(bitmap1.contains(4)); + /// bitmap1.flip_inplace(4..=4); + /// assert_eq!(bitmap1.to_vec(), [1, 2]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_flip_inplace")] + pub fn flip_inplace>(&mut self, range: R) { + let (start, end) = range_to_inclusive(range); + unsafe { ffi::roaring64_bitmap_flip_closed_inplace(self.raw.as_ptr(), start, end) }; + } + + /// Returns a vector containing the values in the bitmap in sorted order + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// assert_eq!(bitmap.to_vec(), vec![1, 2, 3]); + /// ``` + #[must_use] + pub fn to_vec(&self) -> Vec { + let len = self + .cardinality() + .try_into() + .expect("cardinality must fit in a usize"); + + let mut vec = vec![0; len]; + unsafe { ffi::roaring64_bitmap_to_uint64_array(self.raw.as_ptr(), vec.as_mut_ptr()) }; + vec + } + + /// Computes the serialized size in bytes of the Bitmap in format `S`. + #[inline] + #[must_use] + pub fn get_serialized_size_in_bytes(&self) -> usize { + S::get_serialized_size_in_bytes(self) + } + + /// Serializes a bitmap to a slice of bytes in format `S`. + /// + /// # Examples + /// + /// ``` + /// use croaring::{Bitmap64, Portable}; + /// + /// let original_bitmap: Bitmap64 = (1..5).collect(); + /// + /// let serialized_buffer = original_bitmap.serialize::(); + /// + /// let deserialized_bitmap = Bitmap64::deserialize::(&serialized_buffer); + /// + /// assert_eq!(original_bitmap, deserialized_bitmap); + /// ``` + #[inline] + #[must_use] + pub fn serialize(&self) -> Vec { + let mut dst = Vec::new(); + self.serialize_into::(&mut dst); + dst + } + + /// Serializes a bitmap to a slice of bytes in format `S`, re-using existing capacity + /// + /// `dst` is not cleared, data is added after any existing data. Returns the added slice of `dst`. + /// If `dst` is empty, it is guaranteed to hold only the serialized data after this call + /// + /// # Examples + /// + /// ``` + /// use croaring::{Bitmap64, Portable}; + /// + /// let original_bitmap_1: Bitmap64 = (1..5).collect(); + /// let original_bitmap_2: Bitmap64 = (1..10).collect(); + /// + /// let mut data = Vec::new(); + /// for bitmap in [original_bitmap_1, original_bitmap_2] { + /// data.clear(); + /// bitmap.serialize_into::(&mut data); + /// // do something with data + /// } + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_portable_serialize")] + pub fn serialize_into<'a, S: Serializer>(&self, dst: &'a mut Vec) -> &'a [u8] { + S::serialize_into(self, dst) + } + + /// Given a serialized bitmap as slice of bytes in format `S`, returns a `Bitmap64` instance. + /// See example of [`Self::serialize`] function. + /// + /// On invalid input returns None. + /// + /// # Examples + /// + /// ``` + /// use croaring::{Bitmap64, Portable}; + /// + /// let original_bitmap: Bitmap64 = (1..5).collect(); + /// let serialized_buffer = original_bitmap.serialize::(); + /// + /// let deserialized_bitmap = Bitmap64::try_deserialize::(&serialized_buffer); + /// assert_eq!(original_bitmap, deserialized_bitmap.unwrap()); + /// + /// let invalid_buffer: Vec = vec![3]; + /// let deserialized_bitmap = Bitmap64::try_deserialize::(&invalid_buffer); + /// assert!(deserialized_bitmap.is_none()); + /// ``` + #[inline] + #[must_use] + pub fn try_deserialize(buffer: &[u8]) -> Option { + D::try_deserialize(buffer) + } + + /// Given a serialized bitmap as slice of bytes in format `S `, returns a bitmap instance. + /// See example of [`Self::serialize`] function. + /// + /// On invalid input returns empty bitmap. + #[inline] + pub fn deserialize(buffer: &[u8]) -> Self { + Self::try_deserialize::(buffer).unwrap_or_default() + } + + /// Iterate over the values in the bitmap in sorted order + /// + /// If `f` returns `Break`, iteration will stop and the value will be returned, + /// Otherwise, iteration continues. If `f` never returns break, `None` is returned after all values are visited. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// use std::ops::ControlFlow; + /// + /// let bitmap = Bitmap64::of(&[1, 2, 3, 14, 20, 21, 100]); + /// let mut even_nums_under_50 = vec![]; + /// + /// let first_over_50 = bitmap.for_each(|value| { + /// if value > 50 { + /// return ControlFlow::Break(value); + /// } + /// if value % 2 == 0 { + /// even_nums_under_50.push(value); + /// } + /// ControlFlow::Continue(()) + /// }); + /// + /// assert_eq!(even_nums_under_50, vec![2, 14, 20]); + /// assert_eq!(first_over_50, ControlFlow::Break(100)); + /// ``` + #[inline] + pub fn for_each(&self, f: F) -> ControlFlow + where + F: FnMut(u64) -> ControlFlow, + { + let mut callback_wrapper = CallbackWrapper::new(f); + let (callback, context) = callback_wrapper.callback_and_ctx(); + unsafe { + ffi::roaring64_bitmap_iterate(self.raw.as_ptr(), Some(callback), context); + } + match callback_wrapper.result() { + Ok(cf) => cf, + Err(e) => panic::resume_unwind(e), + } + } + + /// Ensure the bitmap is internally valid + /// + /// This is useful for development, but is not needed for normal use: + /// bitmaps should _always_ be internally valid. + /// + /// # Errors + /// + /// Returns an error if the bitmap is not valid, with a description of the problem. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap = Bitmap64::from_range(0..100); + /// bitmap.internal_validate().unwrap(); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_internal_validate")] + #[doc(hidden)] + pub fn internal_validate(&self) -> Result<(), &'static str> { + let mut error_str = ptr::null(); + let valid = + unsafe { ffi::roaring64_bitmap_internal_validate(self.raw.as_ptr(), &mut error_str) }; + if valid { + Ok(()) + } else { + if error_str.is_null() { + return Err("Unknown error"); + } + let reason = unsafe { CStr::from_ptr(error_str) }; + Err(reason.to_str().unwrap_or("Invalid UTF-8")) + } + } +} + +/// Binary Operations +impl Bitmap64 { + /// Return true if self and other contain _any_ common elements + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert!(bitmap1.intersect(&bitmap2)); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_intersect")] + pub fn intersect(&self, other: &Self) -> bool { + unsafe { ffi::roaring64_bitmap_intersect(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Check if a bitmap has any values set in `range` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap = Bitmap64::of(&[1, 100, 101, u64::MAX]); + /// + /// assert!(bitmap.intersect_with_range(0..10)); + /// assert!(!bitmap.intersect_with_range(2..100)); + /// assert!(bitmap.intersect_with_range(999..=u64::MAX)); + /// + /// // Empty ranges + /// assert!(!bitmap.intersect_with_range(100..100)); + /// assert!(!bitmap.intersect_with_range(100..0)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_intersect_with_range")] + pub fn intersect_with_range>(&self, range: R) -> bool { + let Some(exclusive_range) = range_to_exclusive(range) else { + return false; + }; + self._intersect_with_range(exclusive_range) + } + + #[inline] + fn _intersect_with_range(&self, exclusive_range: ExclusiveRangeRes) -> bool { + let ExclusiveRangeRes { + start, + end, + needs_max, + } = exclusive_range; + if needs_max && self.contains(u64::MAX) { + return true; + } + unsafe { ffi::roaring64_bitmap_intersect_with_range(self.raw.as_ptr(), start, end) } + } + + /// Computes the Jaccard index between two bitmaps + /// + /// This is also known as the Tanimoto distance, or the Jaccard similarity coefficient + /// + /// The Jaccard index is NaN if both bitmaps are empty + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert_eq!(bitmap1.jaccard_index(&bitmap2), 0.5); + /// + /// let empty_bitmap = Bitmap64::new(); + /// assert!(empty_bitmap.jaccard_index(&empty_bitmap).is_nan()); + /// ``` + #[inline] + #[must_use] + #[doc(alias = "roaring64_bitmap_jaccard_index")] + pub fn jaccard_index(&self, other: &Self) -> f64 { + unsafe { ffi::roaring64_bitmap_jaccard_index(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the intersection between two bitmaps and returns the result + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// let bitmap3 = bitmap1.and(&bitmap2); + /// assert!(bitmap3.contains(2)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_and")] + #[must_use] + pub fn and(&self, other: &Self) -> Self { + unsafe { + Self::take_heap(ffi::roaring64_bitmap_and( + self.raw.as_ptr(), + other.raw.as_ptr(), + )) + } + } + + /// Computes the size of the intersection between two bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert_eq!(bitmap1.and_cardinality(&bitmap2), 2); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_and_cardinality")] + #[must_use] + pub fn and_cardinality(&self, other: &Self) -> u64 { + unsafe { ffi::roaring64_bitmap_and_cardinality(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the intersection between two bitmaps and stores the result in the current bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// bitmap1.and_inplace(&bitmap2); + /// assert!(bitmap1.contains(2)); + /// assert!(bitmap1.contains(3)); + /// assert!(!bitmap1.contains(1)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_and_inplace")] + pub fn and_inplace(&mut self, other: &Self) { + unsafe { ffi::roaring64_bitmap_and_inplace(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the union between two bitmaps and returns the result + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// let bitmap3 = bitmap1.or(&bitmap2); + /// assert_eq!(bitmap3.to_vec(), vec![1, 2, 3, 4]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_or")] + #[must_use] + pub fn or(&self, other: &Self) -> Self { + unsafe { + Self::take_heap(ffi::roaring64_bitmap_or( + self.raw.as_ptr(), + other.raw.as_ptr(), + )) + } + } + + /// Computes the size of the union between two bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert_eq!(bitmap1.or_cardinality(&bitmap2), 4); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_or_cardinality")] + #[must_use] + pub fn or_cardinality(&self, other: &Self) -> u64 { + unsafe { ffi::roaring64_bitmap_or_cardinality(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the union between two bitmaps and stores the result in the current bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// bitmap1.or_inplace(&bitmap2); + /// assert_eq!(bitmap1.to_vec(), vec![1, 2, 3, 4]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_or_inplace")] + pub fn or_inplace(&mut self, other: &Self) { + unsafe { ffi::roaring64_bitmap_or_inplace(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the symmetric difference (xor) between two bitmaps and returns the result + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// let bitmap3 = bitmap1.xor(&bitmap2); + /// assert_eq!(bitmap3.to_vec(), vec![1, 4]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_xor")] + #[must_use] + pub fn xor(&self, other: &Self) -> Self { + unsafe { + Self::take_heap(ffi::roaring64_bitmap_xor( + self.raw.as_ptr(), + other.raw.as_ptr(), + )) + } + } + + /// Computes the size of the symmetric difference (xor) between two bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert_eq!(bitmap1.xor_cardinality(&bitmap2), 2); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_xor_cardinality")] + #[must_use] + pub fn xor_cardinality(&self, other: &Self) -> u64 { + unsafe { ffi::roaring64_bitmap_xor_cardinality(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the symmetric difference (xor) between two bitmaps and stores the result in the current bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// bitmap1.xor_inplace(&bitmap2); + /// assert_eq!(bitmap1.to_vec(), vec![1, 4]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_xor_inplace")] + pub fn xor_inplace(&mut self, other: &Self) { + unsafe { ffi::roaring64_bitmap_xor_inplace(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the difference between two bitmaps and returns the result + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// let bitmap3 = bitmap1.andnot(&bitmap2); + /// assert_eq!(bitmap3.to_vec(), vec![1]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_andnot")] + #[must_use] + pub fn andnot(&self, other: &Self) -> Self { + unsafe { + Self::take_heap(ffi::roaring64_bitmap_andnot( + self.raw.as_ptr(), + other.raw.as_ptr(), + )) + } + } + + /// Computes the size of the difference between two bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// assert_eq!(bitmap1.andnot_cardinality(&bitmap2), 1); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_andnot_cardinality")] + #[must_use] + pub fn andnot_cardinality(&self, other: &Self) -> u64 { + unsafe { ffi::roaring64_bitmap_andnot_cardinality(self.raw.as_ptr(), other.raw.as_ptr()) } + } + + /// Computes the difference between two bitmaps and stores the result in the current bitmap + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[2, 3, 4]); + /// bitmap1.andnot_inplace(&bitmap2); + /// assert_eq!(bitmap1.to_vec(), vec![1]); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_andnot_inplace")] + pub fn andnot_inplace(&mut self, other: &Self) { + unsafe { ffi::roaring64_bitmap_andnot_inplace(self.raw.as_ptr(), other.raw.as_ptr()) } + } +} + +/// Returns start, end, and whether the range also includes u64::MAX +struct ExclusiveRangeRes { + start: u64, + end: u64, + needs_max: bool, +} + +fn range_to_exclusive>(range: R) -> Option { + let (start, inclusive_end) = range_to_inclusive(range); + + if inclusive_end < start { + return None; + } + + let (end, needs_max) = match inclusive_end.checked_add(1) { + Some(i) => (i, false), + None => (u64::MAX, true), + }; + Some(ExclusiveRangeRes { + start, + end, + needs_max, + }) +} + +fn range_to_inclusive>(range: R) -> (u64, u64) { + let start = match range.start_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => match i.checked_add(1) { + Some(i) => i, + None => return (1, 0), + }, + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i, + Bound::Excluded(&i) => match i.checked_sub(1) { + Some(i) => i, + None => return (1, 0), + }, + Bound::Unbounded => u64::MAX, + }; + (start, end) +} diff --git a/croaring/src/bitmap64/iter.rs b/croaring/src/bitmap64/iter.rs new file mode 100644 index 0000000..7ab39d9 --- /dev/null +++ b/croaring/src/bitmap64/iter.rs @@ -0,0 +1,599 @@ +use crate::Bitmap64; +use std::marker::PhantomData; +use std::mem::{ManuallyDrop, MaybeUninit}; +use std::ptr::NonNull; + +impl FromIterator for Bitmap64 { + /// Convenience method for creating bitmap from iterator. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap: Bitmap64 = (1..3).collect(); + /// + /// assert!(!bitmap.is_empty()); + /// assert!(bitmap.contains(1)); + /// assert!(bitmap.contains(2)); + /// assert_eq!(bitmap.cardinality(), 2); + /// ``` + fn from_iter>(iter: I) -> Self { + let mut bitmap = Bitmap64::new(); + bitmap.extend(iter); + bitmap + } +} + +impl Extend for Bitmap64 { + #[doc(alias = "roaring64_bitmap_add_bulk")] + fn extend>(&mut self, iter: T) { + let mut ctx = MaybeUninit::::zeroed(); + iter.into_iter().for_each(|value| unsafe { + ffi::roaring64_bitmap_add_bulk(self.raw.as_ptr(), ctx.as_mut_ptr(), value); + }); + } +} + +/// A cursor over a bitmap64 +/// +/// A Cursor is like an iterator, except that it can freely seek back-and-forth. +/// +/// A cursor points at a single value in the bitmap, or at a "ghost" position, +/// either one before the beginning of the bitmap, or one after the end of the bitmap. +#[derive(Debug)] +pub struct Bitmap64Cursor<'a> { + raw: NonNull, + has_value: bool, + _bitmap: PhantomData<&'a Bitmap64>, +} + +unsafe impl Send for Bitmap64Cursor<'_> {} + +unsafe impl Sync for Bitmap64Cursor<'_> {} + +impl Drop for Bitmap64Cursor<'_> { + fn drop(&mut self) { + unsafe { + ffi::roaring64_iterator_free(self.raw.as_ptr()); + } + } +} + +impl<'a> Bitmap64Cursor<'a> { + fn from_raw(raw: *mut ffi::roaring64_iterator_t) -> Self { + let raw = NonNull::new(raw).expect("Failed to allocate roaring64_iterator_t"); + let has_value = unsafe { ffi::roaring64_iterator_has_value(raw.as_ptr()) }; + Self { + raw, + has_value, + _bitmap: PhantomData, + } + } + + fn at_first(bitmap: &'a Bitmap64) -> Self { + let raw = unsafe { ffi::roaring64_iterator_create(bitmap.raw.as_ptr()) }; + Self::from_raw(raw) + } + + fn at_last(bitmap: &'a Bitmap64) -> Self { + let raw = unsafe { ffi::roaring64_iterator_create_last(bitmap.raw.as_ptr()) }; + Self::from_raw(raw) + } + + /// Returns true if the cursor is pointing at a value in the bitmap. + /// + /// If this returns false, then the cursor is pointing at a "ghost" position, + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// assert!(!bitmap.cursor().has_value()); + /// + /// bitmap.add(1); + /// let mut cursor = bitmap.cursor(); + /// assert!(cursor.has_value()); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert!(!cursor.has_value()); + /// ``` + #[inline] + pub fn has_value(&self) -> bool { + self.has_value + } + + /// Returns the value at the cursor, if any. + /// + /// If the cursor is not pointing at a value, then this returns `None`. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add(1); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn current(&self) -> Option { + if self.has_value() { + Some(unsafe { self.current_unchecked() }) + } else { + None + } + } + + unsafe fn current_unchecked(&self) -> u64 { + ffi::roaring64_iterator_value(self.raw.as_ptr()) + } + + /// Moves the cursor to the next value in the bitmap + /// + /// If the cursor is already past the end of the bitmap, then this does nothing. + /// + /// If the cursor is at the ghost position before the beginning of the bitmap, + /// then this moves the cursor to the first value in the bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), None); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(2)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn move_next(&mut self) { + self.has_value = unsafe { ffi::roaring64_iterator_advance(self.raw.as_ptr()) }; + } + + /// Moves the cursor to the next value in the bitmap, and returns the value (if any) + /// + /// This is equivalent to calling [`Self::move_next`] followed by [`Self::current`]. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// assert_eq!(cursor.next(), Some(2)); + /// assert_eq!(cursor.next(), Some(3)); + /// assert_eq!(cursor.next(), None); + /// ``` + #[inline] + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> Option { + self.move_next(); + self.current() + } + + /// Moves the cursor to the previous value in the bitmap + /// + /// If the cursor is already before the beginning of the bitmap, then this does nothing. + /// + /// If the cursor is at the ghost position after the end of the bitmap, + /// then this moves the cursor to the last value in the bitmap. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_next(); + /// assert_eq!(cursor.current(), None); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(3)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(2)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), Some(1)); + /// cursor.move_prev(); + /// assert_eq!(cursor.current(), None); + /// ``` + #[inline] + pub fn move_prev(&mut self) { + self.has_value = unsafe { ffi::roaring64_iterator_previous(self.raw.as_ptr()) }; + } + + /// Moves the cursor to the previous value in the bitmap, and returns the value (if any) + /// + /// This is equivalent to calling [`Self::move_prev`] followed by [`Self::current`]. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// let mut cursor = bitmap.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// assert_eq!(cursor.prev(), Some(2)); + /// assert_eq!(cursor.prev(), Some(1)); + /// assert_eq!(cursor.prev(), None); + /// ``` + #[inline] + pub fn prev(&mut self) -> Option { + self.move_prev(); + self.current() + } + + /// Resets this cursor to the first value in the bitmap. + /// + /// The bitmap does not have to be the same bitmap that this cursor was created from: + /// this allows you to reuse a cursor for multiple bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[4, 5, 6]); + /// let cursor = bitmap1.cursor(); + /// assert_eq!(cursor.current(), Some(1)); + /// let cursor = cursor.reset_to_first(&bitmap2); + /// assert_eq!(cursor.current(), Some(4)); + /// // Cursor is no longer borrowing from bitmap1 + /// bitmap1.add(100); + /// ``` + #[must_use] + pub fn reset_to_first(self, bitmap: &Bitmap64) -> Bitmap64Cursor<'_> { + // Don't drop `self` and free the iterator + let this = ManuallyDrop::new(self); + unsafe { ffi::roaring64_iterator_reinit(bitmap.raw.as_ptr(), this.raw.as_ptr()) }; + Bitmap64Cursor::from_raw(this.raw.as_ptr()) + } + + /// Resets this cursor to the last value in the bitmap. + /// + /// The bitmap does not have to be the same bitmap that this cursor was created from: + /// this allows you to reuse a cursor for multiple bitmaps + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap1 = Bitmap64::of(&[1, 2, 3]); + /// let bitmap2 = Bitmap64::of(&[4, 5, 6]); + /// let cursor = bitmap1.cursor_to_last(); + /// assert_eq!(cursor.current(), Some(3)); + /// let cursor = cursor.reset_to_last(&bitmap2); + /// assert_eq!(cursor.current(), Some(6)); + /// ``` + #[must_use] + pub fn reset_to_last(self, bitmap: &Bitmap64) -> Bitmap64Cursor<'_> { + // Don't drop `self` and free the iterator + let this = ManuallyDrop::new(self); + unsafe { ffi::roaring64_iterator_reinit_last(bitmap.raw.as_ptr(), this.raw.as_ptr()) }; + Bitmap64Cursor::from_raw(this.raw.as_ptr()) + } + + /// Attempt to read many values from the iterator into `dst` + /// + /// The current value _is_ included in the output. + /// + /// Returns the number of items read from the iterator, may be `< dst.len()` iff + /// the iterator is exhausted or `dst.len() > u64::MAX`. + /// + /// This can be much more efficient than repeated iteration. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::new(); + /// bitmap.add_range(0..100); + /// bitmap.add(222); + /// bitmap.add(555); + /// bitmap.add(999); + /// + /// let mut buf = [0; 100]; + /// let mut cursor = bitmap.cursor(); + /// assert_eq!(cursor.read_many(&mut buf), 100); + /// // Get the first 100 items, from the original range added + /// for (i, item) in buf.iter().enumerate() { + /// assert_eq!(*item, i as u64); + /// } + /// // Calls to next_many() can be interleaved with other cursor calls + /// assert_eq!(cursor.current(), Some(222)); + /// assert_eq!(cursor.next(), Some(555)); + /// assert_eq!(cursor.read_many(&mut buf), 2); + /// assert_eq!(buf[0], 555); + /// assert_eq!(buf[1], 999); + /// + /// assert_eq!(cursor.current(), None); + /// assert_eq!(cursor.read_many(&mut buf), 0); + /// ``` + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// fn print_by_chunks(bitmap: &Bitmap64) { + /// let mut buf = [0; 1024]; + /// let mut iter = bitmap.cursor(); + /// loop { + /// let n = iter.read_many(&mut buf); + /// if n == 0 { + /// break; + /// } + /// println!("{:?}", &buf[..n]); + /// } + /// } + /// + /// # print_by_chunks(&Bitmap64::of(&[1, 2, 8, 20, 1000])); + /// ``` + #[inline] + #[doc(alias = "roaring64_iterator_read")] + pub fn read_many(&mut self, dst: &mut [u64]) -> usize { + let count = u64::try_from(dst.len()).unwrap_or(u64::MAX); + let result = + unsafe { ffi::roaring64_iterator_read(self.raw.as_ptr(), dst.as_mut_ptr(), count) }; + debug_assert!(result <= count); + self.has_value = unsafe { ffi::roaring64_iterator_has_value(self.raw.as_ptr()) }; + result as usize + } + + /// Reset the iterator to the first value `>= val` + /// + /// This can move the iterator forwards or backwards. + /// + /// # Examples + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap = Bitmap64::of(&[0, 1, 100, 1000, u64::MAX]); + /// let mut cursor = bitmap.cursor(); + /// cursor.reset_at_or_after(0); + /// assert_eq!(cursor.current(), Some(0)); + /// cursor.reset_at_or_after(0); + /// assert_eq!(cursor.current(), Some(0)); + /// + /// cursor.reset_at_or_after(101); + /// assert_eq!(cursor.current(), Some(1000)); + /// assert_eq!(cursor.next(), Some(u64::MAX)); + /// assert_eq!(cursor.next(), None); + /// cursor.reset_at_or_after(u64::MAX); + /// assert_eq!(cursor.current(), Some(u64::MAX)); + /// assert_eq!(cursor.next(), None); + /// ``` + #[inline] + #[doc(alias = "roaring64_iterator_move_equalorlarger")] + pub fn reset_at_or_after(&mut self, val: u64) { + self.has_value = + unsafe { ffi::roaring64_iterator_move_equalorlarger(self.raw.as_ptr(), val) }; + } +} + +/// Converts this iterator into a cursor +/// +/// The cursor's current value will be the the item which would have been returned by the next call to `next()` +/// or one past the end of the bitmap if the iterator is exhausted. +/// +/// # Examples +/// +/// ``` +/// use croaring::bitmap64::{Bitmap64, Bitmap64Cursor}; +/// let mut bitmap = Bitmap64::of(&[1, 2, 3]); +/// let mut iter = bitmap.iter(); +/// assert_eq!(iter.peek(), Some(1)); +/// assert_eq!(iter.next(), Some(1)); +/// +/// assert_eq!(iter.peek(), Some(2)); +/// let mut cursor: Bitmap64Cursor = iter.into(); +/// assert_eq!(cursor.current(), Some(2)); +/// ``` +impl<'a> From> for Bitmap64Cursor<'a> { + fn from(iter: Bitmap64Iterator<'a>) -> Self { + iter.cursor + } +} + +/// Converts this cursor into an iterator +/// +/// The next value returned by the iterator will be the current value of the cursor (if any). +/// +/// # Examples +/// +/// ``` +/// use croaring::bitmap64::{Bitmap64, Bitmap64Iterator}; +/// +/// let mut bitmap = Bitmap64::of(&[1, 2, 3]); +/// let mut cursor = bitmap.cursor(); +/// assert_eq!(cursor.current(), Some(1)); +/// +/// let mut iter = Bitmap64Iterator::from(cursor); +/// assert_eq!(iter.next(), Some(1)); +/// ``` +impl<'a> From> for Bitmap64Iterator<'a> { + fn from(cursor: Bitmap64Cursor<'a>) -> Self { + Bitmap64Iterator { cursor } + } +} + +impl<'a> Clone for Bitmap64Cursor<'a> { + fn clone(&self) -> Self { + let raw = unsafe { ffi::roaring64_iterator_copy(self.raw.as_ptr()) }; + Self::from_raw(raw) + } +} + +/// An iterator over the values in a bitmap +#[derive(Debug, Clone)] +pub struct Bitmap64Iterator<'a> { + cursor: Bitmap64Cursor<'a>, +} + +impl<'a> Bitmap64Iterator<'a> { + fn new(bitmap: &'a Bitmap64) -> Self { + Self { + cursor: bitmap.cursor(), + } + } + + #[inline] + fn advance(&mut self) { + self.cursor.move_next(); + } + + /// Attempt to read many values from the iterator into `dst` + /// + /// Returns the number of items read from the iterator, may be `< dst.len()` iff + /// the iterator is exhausted or `dst.len() > u64::MAX`. + /// + /// This can be much more efficient than repeated iteration. + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap: Bitmap64 = Bitmap64::new(); + /// bitmap.add_range(0..100); + /// bitmap.add(222); + /// bitmap.add(555); + /// + /// let mut buf = [0; 100]; + /// let mut iter = bitmap.iter(); + /// assert_eq!(iter.next_many(&mut buf), 100); + /// // Get the first 100 items, from the original range added + /// for (i, item) in buf.iter().enumerate() { + /// assert_eq!(*item, i as u64); + /// } + /// // Calls to next_many() can be interleaved with calls to next() + /// assert_eq!(iter.next(), Some(222)); + /// assert_eq!(iter.next_many(&mut buf), 1); + /// assert_eq!(buf[0], 555); + /// + /// assert_eq!(iter.next(), None); + /// assert_eq!(iter.next_many(&mut buf), 0); + /// ``` + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// fn print_by_chunks(bitmap: &Bitmap64) { + /// let mut buf = [0; 1024]; + /// let mut iter = bitmap.iter(); + /// loop { + /// let n = iter.next_many(&mut buf); + /// if n == 0 { + /// break; + /// } + /// println!("{:?}", &buf[..n]); + /// } + /// } + /// + /// # print_by_chunks(&Bitmap64::of(&[1, 2, 8, 20, 1000])); + /// ``` + #[inline] + #[doc(alias = "roaring64_iterator_read")] + pub fn next_many(&mut self, dst: &mut [u64]) -> usize { + self.cursor.read_many(dst) + } + + /// Reset the iterator to the first value `>= val` + /// + /// This can move the iterator forwards or backwards. + /// + /// # Examples + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap = Bitmap64::of(&[0, 1, 100, 1000, u64::MAX]); + /// let mut iter = bitmap.iter(); + /// iter.reset_at_or_after(0); + /// assert_eq!(iter.next(), Some(0)); + /// iter.reset_at_or_after(0); + /// assert_eq!(iter.next(), Some(0)); + /// + /// iter.reset_at_or_after(101); + /// assert_eq!(iter.next(), Some(1000)); + /// assert_eq!(iter.next(), Some(u64::MAX)); + /// assert_eq!(iter.next(), None); + /// iter.reset_at_or_after(u64::MAX); + /// assert_eq!(iter.next(), Some(u64::MAX)); + /// assert_eq!(iter.next(), None); + /// ``` + #[inline] + #[doc(alias = "roaring64_iterator_move_equalorlarger")] + pub fn reset_at_or_after(&mut self, val: u64) { + self.cursor.reset_at_or_after(val); + } + + /// Peek at the next value to be returned by the iterator (if any), without consuming it + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// let mut bitmap = Bitmap64::of(&[1, 2, 3]); + /// let mut iter = bitmap.iter(); + /// assert_eq!(iter.peek(), Some(1)); + /// assert_eq!(iter.next(), Some(1)); + /// ``` + #[inline] + pub fn peek(&self) -> Option { + self.cursor.current() + } +} + +impl<'a> Iterator for Bitmap64Iterator<'a> { + type Item = u64; + + #[inline] + fn next(&mut self) -> Option { + match self.peek() { + Some(value) => { + self.advance(); + + Some(value) + } + None => None, + } + } +} + +impl Bitmap64 { + /// Returns an iterator over the values in the bitmap. + #[inline] + #[must_use] + pub fn iter(&self) -> Bitmap64Iterator { + Bitmap64Iterator::new(self) + } + + /// Returns a cursor pointing at the first value in the bitmap. + /// + /// See [`Bitmap64Cursor`] for more details. + #[inline] + #[must_use] + pub fn cursor(&self) -> Bitmap64Cursor { + Bitmap64Cursor::at_first(self) + } + + /// Returns a cursor pointing at the last value in the bitmap. + /// + /// See [`Bitmap64Cursor`] for more details. + #[inline] + #[must_use] + pub fn cursor_to_last(&self) -> Bitmap64Cursor { + Bitmap64Cursor::at_last(self) + } +} diff --git a/croaring/src/bitmap64/mod.rs b/croaring/src/bitmap64/mod.rs new file mode 100644 index 0000000..58d4d9b --- /dev/null +++ b/croaring/src/bitmap64/mod.rs @@ -0,0 +1,17 @@ +//! A compressed bitmap which can hold 64-bit integers + +pub use self::iter::{Bitmap64Cursor, Bitmap64Iterator}; + +mod imp; +mod iter; +mod ops; +mod serialization; + +pub use self::serialization::{Deserializer, Serializer}; + +/// A Bitmap which can hold 64-bit integers +pub struct Bitmap64 { + raw: std::ptr::NonNull, +} +unsafe impl Sync for Bitmap64 {} +unsafe impl Send for Bitmap64 {} diff --git a/croaring/src/bitmap64/ops.rs b/croaring/src/bitmap64/ops.rs new file mode 100644 index 0000000..c49688f --- /dev/null +++ b/croaring/src/bitmap64/ops.rs @@ -0,0 +1,374 @@ +use crate::Bitmap64; +use ffi::roaring64_bitmap_copy; +use std::fmt; +use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Sub, SubAssign}; + +impl fmt::Debug for Bitmap64 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.cardinality() < 32 { + write!(f, "Bitmap64<{:?}>", self.to_vec()) + } else { + write!( + f, + "Bitmap64<{:?} values between {:?} and {:?}>", + self.cardinality(), + self.minimum().unwrap(), + self.maximum().unwrap() + ) + } + } +} + +impl Default for Bitmap64 { + fn default() -> Self { + Self::new() + } +} + +impl From<&'_ [u64]> for Bitmap64 { + #[inline] + #[doc(alias = "roaring64_bitmap_of_ptr")] + fn from(slice: &[u64]) -> Self { + Self::of(slice) + } +} + +impl From<[u64; N]> for Bitmap64 { + #[inline] + #[doc(alias = "roaring64_bitmap_of_ptr")] + fn from(slice: [u64; N]) -> Self { + Self::of(&slice) + } +} + +impl PartialEq for Bitmap64 { + #[inline] + #[doc(alias = "roaring64_bitmap_equals")] + fn eq(&self, other: &Self) -> bool { + unsafe { ffi::roaring64_bitmap_equals(self.raw.as_ptr(), other.raw.as_ptr()) } + } +} + +impl Eq for Bitmap64 {} + +impl Clone for Bitmap64 { + #[inline] + #[doc(alias = "roaring64_bitmap_copy")] + fn clone(&self) -> Self { + unsafe { + let raw = roaring64_bitmap_copy(self.raw.as_ptr()); + Self::take_heap(raw) + } + } +} + +impl Drop for Bitmap64 { + fn drop(&mut self) { + unsafe { + ffi::roaring64_bitmap_free(self.raw.as_ptr()); + } + } +} + +macro_rules! impl_binop { + ( + impl $trait_name:ident { + $(type $type_name:ident = $type_value:ty;)* + + $(#[$($attr:tt)*])* + fn $fn_name:ident -> $ret_ty:ty as $alias:ident + } + ) => { + impl_binop!{ + impl $trait_name { + $(type $type_name = $type_value;)* + + $(#[$($attr)*])* + fn $fn_name(self, other) -> $ret_ty { + self.$alias(&other) + } + } + } + }; + ( + impl $trait_name:ident { + $(type $type_name:ident = $type_value:ty;)* + + $(#[$($attr:tt)*])* + fn $fn_name:ident($self_ident:ident, $other_ident:ident) -> $ret_ty:ty + $body:block + } + ) => { + impl $trait_name for Bitmap64 { + $(type $type_name = $type_value;)* + + $(#[$($attr)*])* + fn $fn_name($self_ident, $other_ident: Bitmap64) -> $ret_ty + $body + } + + impl $trait_name<&Bitmap64> for Bitmap64 { + $(type $type_name = $type_value;)* + + $(#[$($attr)*])* + fn $fn_name($self_ident, $other_ident: &Bitmap64) -> $ret_ty + $body + } + + impl $trait_name for &Bitmap64 { + $(type $type_name = $type_value;)* + + $(#[$($attr)*])* + fn $fn_name($self_ident, $other_ident: Bitmap64) -> $ret_ty + $body + } + + impl $trait_name<&Bitmap64> for &Bitmap64 { + $(type $type_name = $type_value;)* + + $(#[$($attr)*])* + fn $fn_name($self_ident, $other_ident: &Bitmap64) -> $ret_ty + $body + } + }; +} + +macro_rules! impl_binop_assign { + ( + impl $trait_name:ident { + $(#[$($attr:tt)*])* + fn $fn_name:ident as $alias:ident + } + ) => { + impl $trait_name for Bitmap64 { + $(#[$($attr)*])* + fn $fn_name(&mut self, other: Bitmap64) { + self.$alias(&other) + } + } + + impl $trait_name<&'_ Bitmap64> for Bitmap64 { + $(#[$($attr)*])* + fn $fn_name(&mut self, other: &Bitmap64) { + self.$alias(other) + } + } + }; +} + +impl_binop! { + impl BitAnd { + type Output = Bitmap64; + + /// Syntactic sugar for `.and` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::new(); + /// bitmap1.add(1); + /// + /// let mut bitmap2 = Bitmap64::new(); + /// bitmap2.add(1); + /// bitmap2.add(2); + /// + /// let bitmap3 = bitmap1 & bitmap2; + /// + /// assert!(bitmap3.contains(1)); + /// assert!(!bitmap3.contains(2)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_and")] + fn bitand -> Bitmap64 as and + } +} + +impl_binop! { + impl BitOr { + type Output = Bitmap64; + + /// Syntatic sugar for `.or` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1 = Bitmap64::of(&[15]); + /// let bitmap2 = Bitmap64::of(&[25]); + /// + /// let bitmap3 = bitmap1 | bitmap2; + /// + /// assert!(bitmap3.cardinality() == 2); + /// assert!(bitmap3.contains(15)); + /// assert!(bitmap3.contains(25)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_or")] + fn bitor -> Bitmap64 as or + } +} + +impl_binop! { + impl BitXor { + type Output = Bitmap64; + + /// Syntatic sugar for `.xor` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1 = Bitmap64::of(&[15, 25]); + /// let bitmap2 = Bitmap64::of(&[25, 35]); + /// + /// let bitmap3 = bitmap1 ^ bitmap2; + /// + /// assert!(bitmap3.cardinality() == 2); + /// assert!(bitmap3.contains(15)); + /// assert!(!bitmap3.contains(25)); + /// assert!(bitmap3.contains(35)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_xor")] + fn bitxor -> Bitmap64 as xor + } +} + +impl_binop! { + impl Sub { + type Output = Bitmap64; + + /// Syntatic sugar for `.andnot` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let bitmap1 = Bitmap64::of(&[15, 25]); + /// let bitmap2 = Bitmap64::of(&[25, 35]); + /// + /// let bitmap3 = bitmap1 - bitmap2; + /// + /// assert_eq!(bitmap3.cardinality(), 1); + /// assert!(bitmap3.contains(15)); + /// assert!(!bitmap3.contains(25)); + /// assert!(!bitmap3.contains(35)); + /// ``` + #[inline] + #[doc(alias = "andnot")] + #[doc(alias = "roaring64_bitmap_andnot")] + fn sub -> Bitmap64 as andnot + } +} + +impl_binop_assign! { + impl BitAndAssign { + /// Syntactic sugar for `.and_inplace` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::of(&[15]); + /// let bitmap2 = Bitmap64::of(&[25]); + /// let mut bitmap3 = Bitmap64::of(&[15]); + /// let bitmap4 = Bitmap64::of(&[15, 25]); + /// + /// bitmap1 &= bitmap2; + /// + /// assert!(bitmap1.cardinality() == 0); + /// assert!(!bitmap1.contains(15)); + /// assert!(!bitmap1.contains(25)); + /// + /// bitmap3 &= bitmap4; + /// + /// assert!(bitmap3.cardinality() == 1); + /// assert!(bitmap3.contains(15)); + /// assert!(!bitmap3.contains(25)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_and_inplace")] + fn bitand_assign as and_inplace + } +} + +impl_binop_assign! { + impl BitOrAssign { + /// Syntatic sugar for `.or_inplace` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::of(&[15]); + /// let bitmap2 = Bitmap64::of(&[25]); + /// + /// bitmap1 |= bitmap2; + /// + /// assert!(bitmap1.cardinality() == 2); + /// assert!(bitmap1.contains(15)); + /// assert!(bitmap1.contains(25)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_or_inplace")] + fn bitor_assign as or_inplace + } +} + +impl_binop_assign! { + impl BitXorAssign { + /// Syntatic sugar for `.xor_inplace` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::of(&[15, 25]); + /// let bitmap2 = Bitmap64::of(&[25, 35]); + /// + /// bitmap1 ^= bitmap2; + /// + /// assert!(bitmap1.cardinality() == 2); + /// assert!(bitmap1.contains(15)); + /// assert!(!bitmap1.contains(25)); + /// assert!(bitmap1.contains(35)); + /// ``` + #[inline] + #[doc(alias = "roaring64_bitmap_xor_inplace")] + fn bitxor_assign as xor_inplace + } +} + +impl_binop_assign! { + impl SubAssign { + /// Syntatic sugar for `.andnot_inplace` + /// + /// # Examples + /// + /// ``` + /// use croaring::Bitmap64; + /// + /// let mut bitmap1 = Bitmap64::of(&[15, 25]); + /// let bitmap2 = Bitmap64::of(&[25, 35]); + /// + /// bitmap1 -= bitmap2; + /// + /// assert_eq!(bitmap1.cardinality(), 1); + /// assert!(bitmap1.contains(15)); + /// assert!(!bitmap1.contains(25)); + /// assert!(!bitmap1.contains(35)); + /// ``` + #[inline] + #[doc(alias = "andnot_inplace")] + #[doc(alias = "roaring64_bitmap_andnot_inplace")] + fn sub_assign as andnot_inplace + } +} diff --git a/croaring/src/bitmap64/serialization.rs b/croaring/src/bitmap64/serialization.rs new file mode 100644 index 0000000..f03188c --- /dev/null +++ b/croaring/src/bitmap64/serialization.rs @@ -0,0 +1,119 @@ +use crate::{Bitmap64, Portable}; +use std::ffi::c_char; +use std::num::NonZeroUsize; + +/// Trait for different formats of bitmap64 serialization +pub trait Serializer { + /// Serialize a bitmap into bytes, using the provided vec buffer to store the serialized data + /// + /// Note that some serializers ([Frozen][crate::Frozen]) may require that the + /// bitmap is aligned specially, this method will ensure that the returned slice of bytes is + /// aligned correctly, by adding additional padding before the serialized data if required. + /// + /// The contents of the provided vec buffer will not be overwritten: only new data will be + /// appended to the end of the buffer. If the buffer has enough capacity, and the current + /// end of the buffer is correctly aligned, then no additional allocations will be performed. + fn serialize_into<'a>(bitmap: &Bitmap64, dst: &'a mut Vec) -> &'a [u8]; + /// Get the number of bytes required to serialize this bitmap + /// + /// This does not include any additional padding which may be required to align the bitmap + fn get_serialized_size_in_bytes(bitmap: &Bitmap64) -> usize; +} + +/// Trait for different formats of bitmap deserialization +pub trait Deserializer { + /// Try to deserialize a bitmap from the beginning of the provided buffer + /// + /// The [`Bitmap64::try_deserialize`] method should usually be used instead of this method + /// directly. + /// + /// If the buffer starts with the serialized representation of a bitmap, then + /// this method will return a new bitmap containing the deserialized data. + /// + /// If the buffer does not start with a serialized bitmap (or contains an invalidly + /// truncated bitmap), then this method will return `None`. + /// + /// To determine how many bytes were consumed from the buffer, use the + /// [`Serializer::get_serialized_size_in_bytes`] method on the returned bitmap. + fn try_deserialize(buffer: &[u8]) -> Option; + + /// Deserialize a bitmap from the beginning of the provided buffer + /// + /// # Safety + /// + /// Unlike its safe counterpart, [`Self::try_deserialize`], this function assumes the data is valid, + /// passing data which does not contain/start with a bitmap serialized with this format will + /// result in undefined behavior. + unsafe fn try_deserialize_unchecked(buffer: &[u8]) -> Bitmap64; + + /// Find the end of a serialized bitmap in portable format + /// + /// Returns the number of bytes in the buffer which are part of the serialized bitmap, or `None` if + /// the buffer does not start with a valid serialized bitmap. + fn find_end(buffer: &[u8]) -> Option; +} + +impl Serializer for Portable { + /// Serialize a bitmap to a slice of bytes in portable format. + /// + /// See [`Bitmap64::serialize_into`] for more details. + #[doc(alias = "roaring64_bitmap_portable_serialize")] + fn serialize_into<'a>(bitmap: &Bitmap64, dst: &'a mut Vec) -> &'a [u8] { + let len = Self::get_serialized_size_in_bytes(bitmap); + + dst.reserve(len); + let offset = dst.len(); + let total_len = offset.checked_add(len).unwrap(); + + unsafe { + ffi::roaring64_bitmap_portable_serialize( + bitmap.raw.as_ptr(), + dst.spare_capacity_mut().as_mut_ptr().cast::(), + ); + dst.set_len(total_len); + } + &dst[offset..] + } + + /// Computes the serialized size in bytes of the Bitmap in portable format. + /// See [`Bitmap64::get_serialized_size_in_bytes`] for examples. + #[doc(alias = "roaring64_bitmap_portable_size_in_bytes")] + fn get_serialized_size_in_bytes(bitmap: &Bitmap64) -> usize { + unsafe { ffi::roaring64_bitmap_portable_size_in_bytes(bitmap.raw.as_ptr()) } + } +} + +impl Deserializer for Portable { + #[doc(alias = "roaring64_bitmap_portable_deserialize_safe")] + fn try_deserialize(buffer: &[u8]) -> Option { + let raw = unsafe { + ffi::roaring64_bitmap_portable_deserialize_safe(buffer.as_ptr().cast(), buffer.len()) + }; + if raw.is_null() { + return None; + } + + unsafe { + let bitmap = Bitmap64::take_heap(raw); + if bitmap.internal_validate().is_ok() { + Some(bitmap) + } else { + None + } + } + } + + unsafe fn try_deserialize_unchecked(buffer: &[u8]) -> Bitmap64 { + Self::try_deserialize(buffer).unwrap_unchecked() + } + + fn find_end(buffer: &[u8]) -> Option { + let end = unsafe { + ffi::roaring64_bitmap_portable_deserialize_size( + buffer.as_ptr().cast::(), + buffer.len(), + ) + }; + NonZeroUsize::new(end) + } +} diff --git a/croaring/src/bitset/imp.rs b/croaring/src/bitset/imp.rs index e42d3ad..3e3137d 100644 --- a/croaring/src/bitset/imp.rs +++ b/croaring/src/bitset/imp.rs @@ -9,7 +9,7 @@ impl Bitset { let result = Self { bitset: p.read() }; // It seems unlikely that the bitset type will meaningfully change, but check if we ever go // to a version 3. - const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 2); + const _: () = assert!(ffi::ROARING_VERSION_MAJOR == 3); ffi::roaring_free(p.cast()); result } diff --git a/croaring/src/callback.rs b/croaring/src/callback.rs new file mode 100644 index 0000000..7ce0434 --- /dev/null +++ b/croaring/src/callback.rs @@ -0,0 +1,56 @@ +use std::any::Any; +use std::ops::ControlFlow; +use std::panic::AssertUnwindSafe; +use std::{panic, ptr}; + +pub struct CallbackWrapper { + f: F, + result: Result, Box>, +} + +impl CallbackWrapper { + pub fn new(f: F) -> Self { + Self { + f, + result: Ok(ControlFlow::Continue(())), + } + } + + unsafe extern "C" fn raw_callback(value: I, arg: *mut std::ffi::c_void) -> bool + where + I: panic::UnwindSafe, + F: FnMut(I) -> ControlFlow, + { + let wrapper = &mut *(arg as *mut Self); + let mut f = AssertUnwindSafe(&mut wrapper.f); + let result = panic::catch_unwind(move || f(value)); + match result { + Ok(ControlFlow::Continue(())) => true, + Ok(cf @ ControlFlow::Break(_)) => { + wrapper.result = Ok(cf); + false + } + Err(err) => { + wrapper.result = Err(err); + false + } + } + } + + pub fn callback_and_ctx( + &mut self, + ) -> ( + unsafe extern "C" fn(I, *mut std::ffi::c_void) -> bool, + *mut std::ffi::c_void, + ) + where + I: panic::UnwindSafe, + F: FnMut(I) -> ControlFlow, + { + (Self::raw_callback::, ptr::addr_of_mut!(*self).cast()) + } + + pub fn result(self) -> Result, Box> { + self.result + } +} diff --git a/croaring/src/lib.rs b/croaring/src/lib.rs index 680262d..52b6fe1 100644 --- a/croaring/src/lib.rs +++ b/croaring/src/lib.rs @@ -4,14 +4,17 @@ //! Provides Compressed Bitmaps, which act like a set of integers in an efficient way. pub mod bitmap; +pub mod bitmap64; pub mod bitset; pub mod treemap; +mod callback; mod serialization; pub use serialization::*; pub use bitmap::Bitmap; +pub use bitmap64::Bitmap64; pub use bitset::Bitset; pub use treemap::Treemap; diff --git a/croaring/src/treemap/imp.rs b/croaring/src/treemap/imp.rs index c78f3e9..a4d9d69 100644 --- a/croaring/src/treemap/imp.rs +++ b/croaring/src/treemap/imp.rs @@ -3,6 +3,7 @@ use crate::Treemap; use super::util; use crate::treemap::{Deserializer, Serializer}; +use std::cmp::Ordering; use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::ops::{Bound, RangeBounds}; @@ -1220,19 +1221,23 @@ where loop { let (key, bitmap) = match (lhs_next, rhs_next) { (Some((&lhs_key, lhs_bitmap)), Some((&rhs_key, rhs_bitmap))) => { - if lhs_key == rhs_key { - let bitmap = f(BinopArgs::Both(lhs_bitmap, rhs_bitmap)); - lhs_next = lhs_iter.next(); - rhs_next = rhs_iter.next(); - (lhs_key, bitmap) - } else if lhs_key < rhs_key { - let bitmap = f(BinopArgs::Lhs(lhs_bitmap)); - lhs_next = lhs_iter.next(); - (lhs_key, bitmap) - } else { - let bitmap = f(BinopArgs::Rhs(rhs_bitmap)); - rhs_next = rhs_iter.next(); - (rhs_key, bitmap) + match lhs_key.cmp(&rhs_key) { + Ordering::Equal => { + let bitmap = f(BinopArgs::Both(lhs_bitmap, rhs_bitmap)); + lhs_next = lhs_iter.next(); + rhs_next = rhs_iter.next(); + (lhs_key, bitmap) + } + Ordering::Less => { + let bitmap = f(BinopArgs::Lhs(lhs_bitmap)); + lhs_next = lhs_iter.next(); + (lhs_key, bitmap) + } + Ordering::Greater => { + let bitmap = f(BinopArgs::Rhs(rhs_bitmap)); + rhs_next = rhs_iter.next(); + (rhs_key, bitmap) + } } } (Some((&lhs_key, lhs_bitmap)), None) => { diff --git a/croaring/src/treemap/iter.rs b/croaring/src/treemap/iter.rs index d25504a..7ea7fe3 100644 --- a/croaring/src/treemap/iter.rs +++ b/croaring/src/treemap/iter.rs @@ -2,7 +2,7 @@ use super::util; use crate::bitmap::BitmapIterator; use crate::{Bitmap, Treemap}; use std::collections::btree_map; -use std::iter::{self, FromIterator}; +use std::iter; struct To64Iter<'a> { key: u32, @@ -17,10 +17,11 @@ impl<'a> Iterator for To64Iter<'a> { } } -fn to64iter<'a>(t: (&'a u32, &'a Bitmap)) -> To64Iter<'a> { +fn to64iter<'a>((key, bitmap): (&'a u32, &'a Bitmap)) -> To64Iter<'a> { + assert!(!bitmap.is_empty(), "empty bitmap at {key}"); To64Iter { - key: *t.0, - iterator: t.1.iter(), + key: *key, + iterator: bitmap.iter(), } } diff --git a/croaring/tests/data/create_serialization.c b/croaring/tests/data/create_serialization.c index 5732760..88fe867 100644 --- a/croaring/tests/data/create_serialization.c +++ b/croaring/tests/data/create_serialization.c @@ -1,5 +1,6 @@ #include #include +#include #include void write_file(const char *path, const char *contents, size_t len) { @@ -34,7 +35,15 @@ void write_native(const roaring_bitmap_t *b) { roaring_free(data); } -int main(void) { +void write_portable64(const roaring64_bitmap_t *b) { + size_t size = roaring64_bitmap_portable_size_in_bytes(b); + char *data = roaring_malloc(size); + roaring64_bitmap_portable_serialize(b, data); + write_file("portable_bitmap64.bin", data, size); + roaring_free(data); +} + +roaring_bitmap_t *make_bitmap(void) { int i; roaring_bitmap_t *b = roaring_bitmap_create(); @@ -51,9 +60,43 @@ int main(void) { roaring_bitmap_run_optimize(b); + return b; +} + +roaring64_bitmap_t *make_bitmap64(void) { + int i; + int j; + uint64_t base; + + roaring64_bitmap_t *b = roaring64_bitmap_create(); + + for (i = 0; i < 2; ++i) { + base = (uint64_t)i << 32; + // Range container + roaring64_bitmap_add_range_closed(b, base | 0x00000, base | 0x09000); + roaring64_bitmap_add_range_closed(b, base | 0x0A000, base | 0x10000); + // Array container + roaring64_bitmap_add(b, base | 0x20000); + roaring64_bitmap_add(b, base | 0x20005); + // Bitmap container + for (j = 0; j < 0x10000; j += 2) { + roaring64_bitmap_add(b, base | 0x80000 + j); + } + } + + roaring64_bitmap_run_optimize(b); + + return b; +} + +int main(void) { + roaring_bitmap_t *b = make_bitmap(); write_frozen(b); write_portable(b); write_native(b); - roaring_bitmap_free(b); + + roaring64_bitmap_t *b64 = make_bitmap64(); + write_portable64(b64); + roaring64_bitmap_free(b64); } \ No newline at end of file diff --git a/croaring/tests/data/portable_bitmap64.bin b/croaring/tests/data/portable_bitmap64.bin new file mode 100644 index 0000000..acd0f90 Binary files /dev/null and b/croaring/tests/data/portable_bitmap64.bin differ diff --git a/croaring/tests/lib.rs b/croaring/tests/lib.rs index 239bfcb..f0b8fe9 100644 --- a/croaring/tests/lib.rs +++ b/croaring/tests/lib.rs @@ -102,6 +102,34 @@ fn expected_serialized_bitmap() -> Bitmap { bitmap } +#[test] +fn empty_cursor() { + let bitmap = Bitmap::new(); + let mut cursor = bitmap.cursor(); + assert!(!cursor.has_value()); + assert_eq!(cursor.current(), None); + assert_eq!(cursor.prev(), None); + assert_eq!(cursor.prev(), None); + assert_eq!(cursor.next(), None); + assert_eq!(cursor.next(), None); +} + +#[test] +fn cursor_return_from_the_edge() { + let bitmap = Bitmap::from([1, 2, u32::MAX]); + let mut cursor = bitmap.cursor_to_last(); + assert_eq!(cursor.current(), Some(u32::MAX)); + assert_eq!(cursor.next(), None); + assert_eq!(cursor.prev(), Some(u32::MAX)); + assert_eq!(cursor.prev(), Some(2)); + assert_eq!(cursor.prev(), Some(1)); + + assert_eq!(cursor.current(), Some(1)); + assert_eq!(cursor.prev(), None); + assert_eq!(cursor.prev(), None); + assert_eq!(cursor.next(), Some(1)); +} + #[test] fn test_portable_view() { let buffer = fs::read("tests/data/portable_bitmap.bin").unwrap(); diff --git a/croaring/tests/roaring64.rs b/croaring/tests/roaring64.rs new file mode 100644 index 0000000..abcef9e --- /dev/null +++ b/croaring/tests/roaring64.rs @@ -0,0 +1,57 @@ +use croaring::{Bitmap64, Portable}; +use std::fs; + +fn expected_serialized_bitmap() -> Bitmap64 { + let mut bitmap = Bitmap64::new(); + + for i in 0..2u64 { + let base = i << 32; + // Range container + bitmap.add_range(base | 0x0_0000..=base | 0x0_9000); + bitmap.add_range(base | 0x0_A000..=base | 0x1_0000); + // Array container + bitmap.add(base | 0x2_0000); + bitmap.add(base | 0x2_0005); + // Bitmap container + for j in (0..0x1_0000).step_by(2) { + bitmap.add(base | 0x80000 + j); + } + } + bitmap +} + +#[test] +fn test_portable_deserialize() { + let buffer = fs::read("tests/data/portable_bitmap64.bin").unwrap(); + let bitmap = Bitmap64::deserialize::(&buffer); + let expected = expected_serialized_bitmap(); + assert_eq!(bitmap, expected); + assert!(bitmap.iter().eq(expected.iter())) +} + +#[test] +fn test_r64_contains_max() { + let mut bitmap = Bitmap64::new(); + assert!(!bitmap.contains_range((u64::MAX - 1)..=u64::MAX)); + bitmap.add(u64::MAX - 1); + bitmap.add(u64::MAX); + assert!(bitmap.contains_range((u64::MAX - 1)..=u64::MAX)); +} + +#[test] +fn test_r64_cursor_reset() { + let bitmap = Bitmap64::of(&[0, 1, 100, 1000, u64::MAX]); + let mut cursor = bitmap.cursor(); + cursor.reset_at_or_after(0); + assert_eq!(cursor.current(), Some(0)); + cursor.reset_at_or_after(0); + assert_eq!(cursor.current(), Some(0)); + + cursor.reset_at_or_after(101); + assert_eq!(cursor.current(), Some(1000)); + assert_eq!(cursor.next(), Some(u64::MAX)); + assert_eq!(cursor.next(), None); + cursor.reset_at_or_after(u64::MAX); + assert_eq!(cursor.current(), Some(u64::MAX)); + assert_eq!(cursor.next(), None); +} diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 8532d7a..56e9e7e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -27,6 +27,12 @@ path = "fuzz_targets/fuzz_ops.rs" test = false doc = false +[[bin]] +name = "fuzz_ops64" +path = "fuzz_targets/fuzz_ops64.rs" +test = false +doc = false + [[bin]] name = "against_bitvec" path = "fuzz_targets/against_bitvec.rs" diff --git a/fuzz/fuzz_targets/arbitrary_ops64/mod.rs b/fuzz/fuzz_targets/arbitrary_ops64/mod.rs new file mode 100644 index 0000000..ae845b3 --- /dev/null +++ b/fuzz/fuzz_targets/arbitrary_ops64/mod.rs @@ -0,0 +1,497 @@ +use croaring::bitmap64::Bitmap64Cursor; +use croaring::{Bitmap64, Portable, Treemap}; +use libfuzzer_sys::arbitrary::{self, Arbitrary, Unstructured}; +use std::mem; +use std::ops::RangeInclusive; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct Num(pub u64); + +pub const MAX_NUM: u64 = 0x1_0000 * 260; + +impl<'a> Arbitrary<'a> for Num { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + Ok(Self(u.int_in_range(0..=(MAX_NUM - 1))?)) + } +} + +#[derive(Arbitrary, Debug, PartialEq, Eq)] +pub enum MutableBitmapOperation { + Add(Num), + AddChecked(Num), + AddMany(Vec), + AddRange(RangeInclusive), + RemoveRange(RangeInclusive), + FlipRange(RangeInclusive), + Copy, + Clear, + Remove(Num), + RemoveChecked(Num), + RunOptimize, + RemoveRunCompression, + MakeDeep, + MakeWide, + // Add to the max key (or with 0xFFFFFFFF_FFFF0000) + AddToMax(u16), +} + +#[derive(Arbitrary, Debug, PartialEq, Eq)] +pub enum MutableRhsBitmapOperation { + MutateSelf(MutableBitmapOperation), + MutBinaryOp(BitmapMutBinop), +} + +#[derive(Arbitrary, Debug, PartialEq, Eq)] +pub enum BitmapMutBinop { + And, + Or, + Xor, + AndNot, +} + +impl BitmapMutBinop { + fn on_treemap(&self, lhs: &mut Treemap, rhs: &Treemap) { + match *self { + BitmapMutBinop::And => { + lhs.and_inplace(rhs); + } + BitmapMutBinop::Or => { + lhs.or_inplace(rhs); + } + BitmapMutBinop::Xor => { + lhs.xor_inplace(rhs); + } + BitmapMutBinop::AndNot => { + lhs.andnot_inplace(rhs); + } + } + } + + fn on_roaring64(&self, lhs: &mut Bitmap64, rhs: &Bitmap64) { + match *self { + BitmapMutBinop::And => { + let expected = lhs.and(rhs); + lhs.and_inplace(rhs); + assert_eq!(expected, *lhs); + } + BitmapMutBinop::Or => { + let expected = lhs.or(rhs); + lhs.or_inplace(rhs); + assert_eq!(expected, *lhs); + } + BitmapMutBinop::Xor => { + let expected = lhs.xor(rhs); + lhs.xor_inplace(rhs); + assert_eq!(expected, *lhs); + } + BitmapMutBinop::AndNot => { + let expected = lhs.andnot(rhs); + lhs.andnot_inplace(rhs); + assert_eq!(expected, *lhs); + } + } + } +} + +#[derive(Arbitrary, Debug, PartialEq, Eq, Hash)] +pub enum BitmapCompOperation { + Eq, + IsSubset, + IsStrictSubset, + Intersect, + JacardIndex, + And, + Or, + Xor, + AndNot, +} + +#[derive(Arbitrary, Debug, Copy, Clone, PartialEq, Eq)] +pub enum IteratorOp { + HasValue, + Current, + MoveNext, + Next, + MovePrev, + Prev, + ResetToFirst, + ResetToLast, + Clone, + ReadMany(Num), + ResetAtOrAfter(u64), +} + +impl IteratorOp { + pub fn on_cursor<'a>(self, bitmap: &'a Bitmap64, cursor: &mut Bitmap64Cursor<'a>) { + match self { + IteratorOp::HasValue => { + _ = cursor.has_value(); + } + IteratorOp::Current => { + _ = cursor.current(); + } + IteratorOp::MoveNext => { + _ = cursor.move_next(); + } + IteratorOp::Next => { + let v = cursor.next(); + assert_eq!(v, cursor.current()); + } + IteratorOp::MovePrev => { + _ = cursor.move_prev(); + } + IteratorOp::Prev => { + let v = cursor.prev(); + assert_eq!(v, cursor.current()); + } + IteratorOp::ResetToFirst => *cursor = cursor.clone().reset_to_first(bitmap), + IteratorOp::ResetToLast => *cursor = cursor.clone().reset_to_last(bitmap), + IteratorOp::ReadMany(Num(n)) => { + let mut dst = vec![0; n as usize]; + cursor.read_many(&mut dst); + } + IteratorOp::ResetAtOrAfter(n) => { + cursor.reset_at_or_after(n); + } + IteratorOp::Clone => { + *cursor = cursor.clone(); + } + } + } +} + +#[derive(Arbitrary, Debug, PartialEq, Eq, Hash)] +pub enum ReadBitmapOp { + ContainsRange(RangeInclusive), + Contains(u64), + RangeCardinality(RangeInclusive), + Cardinality, + ToVec, + GetPortableSerializedSizeInBytes, + PortableSerialize, + IsEmpty, + IntersectWithRange(RangeInclusive), + Minimum, + Maximum, + Rank(u64), + Index(u64), + Select(u64), + Clone, + Debug, +} + +impl ReadBitmapOp { + pub fn check_against_tree(&self, b: &Bitmap64, t: &Treemap) { + match *self { + ReadBitmapOp::Contains(i) => { + assert_eq!(b.contains(i), t.contains(i)); + } + ReadBitmapOp::RangeCardinality(ref r) => { + // Tree doesn't implement directly, but we can do it manually + let mut t_with_range = t.clone(); + if !r.is_empty() { + t_with_range.remove_range(0..*r.start()); + if let Some(after_end) = r.end().checked_add(1) { + t_with_range.remove_range(after_end..); + } + } + assert_eq!( + b.range_cardinality(r.start()..=r.end()), + t_with_range.cardinality() + ); + } + ReadBitmapOp::Cardinality => { + assert_eq!(b.cardinality(), t.cardinality()); + } + ReadBitmapOp::IsEmpty => { + assert_eq!(b.is_empty(), b.cardinality() == 0); + assert_eq!(b.is_empty(), t.is_empty()); + } + ReadBitmapOp::Minimum => { + assert_eq!(b.minimum(), t.minimum()); + } + ReadBitmapOp::Maximum => { + assert_eq!(b.maximum(), t.maximum()); + } + ReadBitmapOp::Rank(i) => { + assert_eq!(b.rank(i), t.rank(i)); + } + ReadBitmapOp::Index(i) => { + assert_eq!(b.position(i), t.position(i)); + } + ReadBitmapOp::Select(i) => { + assert_eq!(b.select(i), t.select(i)); + } + ReadBitmapOp::Clone => { + let other = b.clone(); + assert_eq!(b, &other); + } + ReadBitmapOp::Debug => { + use std::io::Write; + let mut black_hole = std::io::sink(); + write!(black_hole, "{:?}", b).unwrap(); + } + ReadBitmapOp::ToVec => { + assert_eq!(b.to_vec(), t.to_vec()); + } + ReadBitmapOp::GetPortableSerializedSizeInBytes => { + assert_eq!( + b.get_serialized_size_in_bytes::(), + t.get_serialized_size_in_bytes::() + ); + } + ReadBitmapOp::PortableSerialize => { + assert_eq!(b.serialize::(), t.serialize::(),) + } + ReadBitmapOp::ContainsRange(ref range) => { + // Unsupported by treemaps + _ = b.contains_range(range.start()..=range.end()); + } + ReadBitmapOp::IntersectWithRange(ref range) => { + // Unsupported by treemaps + _ = b.intersect_with_range(range.start()..=range.end()); + } + } + } +} + +impl MutableBitmapOperation { + pub fn on_treemap(&self, t: &mut Treemap) { + match *self { + MutableBitmapOperation::Add(Num(i)) => { + t.add(i); + } + MutableBitmapOperation::AddChecked(Num(i)) => { + let expected = !t.contains(i); + let result = t.add_checked(i); + assert_eq!(expected, result); + } + MutableBitmapOperation::AddMany(ref items) => { + for &Num(item) in items { + t.add(item) + } + } + MutableBitmapOperation::AddRange(ref r) => { + t.add_range(r.start().0..=r.end().0); + } + MutableBitmapOperation::RemoveRange(ref r) => { + t.remove_range(r.start().0..=r.end().0); + } + MutableBitmapOperation::Clear => { + t.clear(); + } + MutableBitmapOperation::Remove(Num(i)) => { + t.remove(i); + } + MutableBitmapOperation::RemoveChecked(Num(i)) => { + let expected = t.contains(i); + let result = t.remove_checked(i); + assert_eq!(expected, result); + } + MutableBitmapOperation::RunOptimize => { + t.run_optimize(); + } + MutableBitmapOperation::RemoveRunCompression => { + // TODO: For now, we don't support removing run compression on roaring64, so + // we don't do it for treemaps (so we can compare how they serialize) + // t.remove_run_compression(); + } + MutableBitmapOperation::Copy => { + *t = t.clone(); + } + MutableBitmapOperation::AddToMax(low_bits) => { + const UPPER_BITS: u64 = 0xFFFF_FFFF_FFFF_0000; + t.add(UPPER_BITS | u64::from(low_bits)); + } + MutableBitmapOperation::FlipRange(ref range) => { + // Treemap's flip is inplace + () = t.flip(range.start().0..=range.end().0); + } + MutableBitmapOperation::MakeDeep => { + t.add(0); + for i in 0..6 { + let val = 1u64 << (i * 8 + 16); + t.add(val); + } + } + MutableBitmapOperation::MakeWide => { + for i in 0..200 { + t.add(i * 0x1_0000); + } + } + } + } + + pub fn on_bitmap64(&self, b: &mut Bitmap64) { + match *self { + MutableBitmapOperation::Add(Num(i)) => { + b.add(i); + } + MutableBitmapOperation::AddChecked(Num(i)) => { + let expected = !b.contains(i); + let result = b.add_checked(i); + assert_eq!(expected, result); + } + MutableBitmapOperation::AddMany(ref items) => { + let items: &[u64] = unsafe { mem::transmute(&items[..]) }; + b.add_many(items); + } + MutableBitmapOperation::AddRange(ref range) => { + b.add_range(range.start().0..=range.end().0); + } + MutableBitmapOperation::RemoveRange(ref range) => { + b.remove_range(range.start().0..=range.end().0); + } + MutableBitmapOperation::Copy => { + *b = b.clone(); + } + MutableBitmapOperation::Clear => { + b.remove_range(..); + } + MutableBitmapOperation::Remove(Num(i)) => { + b.remove(i); + } + MutableBitmapOperation::RemoveChecked(Num(i)) => { + let expected = b.contains(i); + let result = b.remove_checked(i); + assert_eq!(expected, result); + } + MutableBitmapOperation::RunOptimize => { + b.run_optimize(); + } + MutableBitmapOperation::RemoveRunCompression => { + // Unsupported + } + MutableBitmapOperation::AddToMax(low_bits) => { + const UPPER_BITS: u64 = 0xFFFF_FFFF_FFFF_0000; + b.add(UPPER_BITS | u64::from(low_bits)); + } + MutableBitmapOperation::FlipRange(ref range) => { + let expected = b.flip(range.start().0..=range.end().0); + b.flip_inplace(range.start().0..=range.end().0); + assert_eq!(expected, *b); + } + MutableBitmapOperation::MakeDeep => { + b.add(0); + for i in 0..6 { + let val = 1u64 << (i * 8 + 16); + b.add(val); + } + } + MutableBitmapOperation::MakeWide => { + for i in 0..200 { + b.add(i * 0x1_0000); + } + } + } + } +} + +impl MutableRhsBitmapOperation { + pub fn on_treemap(&self, current: &mut Treemap, other: &Treemap) { + match *self { + MutableRhsBitmapOperation::MutateSelf(ref op) => { + op.on_treemap(current); + } + MutableRhsBitmapOperation::MutBinaryOp(ref op) => { + op.on_treemap(current, other); + } + } + } + + pub fn on_bitmap64(&self, current: &mut Bitmap64, other: &Bitmap64) { + match *self { + MutableRhsBitmapOperation::MutateSelf(ref op) => { + op.on_bitmap64(current); + } + MutableRhsBitmapOperation::MutBinaryOp(ref op) => { + op.on_roaring64(current, other); + } + } + } +} + +impl BitmapCompOperation { + pub fn compare_with_tree( + &self, + lhs_bitmap: &Bitmap64, + rhs_bitmap: &Bitmap64, + lhs_tree: &Treemap, + rhs_tree: &Treemap, + ) { + match *self { + BitmapCompOperation::Eq => { + assert_eq!(lhs_bitmap == rhs_bitmap, lhs_tree == rhs_tree); + } + BitmapCompOperation::IsSubset => { + assert_eq!( + lhs_bitmap.is_subset(rhs_bitmap), + lhs_tree.is_subset(rhs_tree), + ); + } + BitmapCompOperation::IsStrictSubset => { + assert_eq!( + lhs_bitmap.is_strict_subset(rhs_bitmap), + lhs_tree.is_strict_subset(rhs_tree), + ); + } + BitmapCompOperation::Intersect => { + let tree_intersect = !(lhs_tree & rhs_tree).is_empty(); + assert_eq!(lhs_bitmap.intersect(rhs_bitmap), tree_intersect); + assert!(lhs_bitmap.is_empty() || lhs_bitmap.intersect(lhs_bitmap)); + } + BitmapCompOperation::JacardIndex => { + // Treemap doesn't support jaccard index + _ = lhs_bitmap.jaccard_index(rhs_bitmap); + _ = lhs_bitmap.jaccard_index(lhs_bitmap); + } + BitmapCompOperation::And => { + let bitmap = lhs_bitmap & rhs_bitmap; + let treemap = lhs_tree & rhs_tree; + + assert_64_eq(&bitmap, &treemap); + assert_eq!(bitmap.cardinality(), lhs_bitmap.and_cardinality(rhs_bitmap)); + } + BitmapCompOperation::Or => { + let bitmap = lhs_bitmap | rhs_bitmap; + let treemap = lhs_tree | rhs_tree; + + assert_64_eq(&bitmap, &treemap); + assert_eq!(bitmap.cardinality(), lhs_bitmap.or_cardinality(rhs_bitmap)); + } + BitmapCompOperation::Xor => { + let bitmap = lhs_bitmap ^ rhs_bitmap; + let treemap = lhs_tree ^ rhs_tree; + + assert_64_eq(&bitmap, &treemap); + assert_eq!(bitmap.cardinality(), lhs_bitmap.xor_cardinality(rhs_bitmap)); + } + BitmapCompOperation::AndNot => { + let bitmap = lhs_bitmap - rhs_bitmap; + let treemap = lhs_tree - rhs_tree; + + assert_64_eq(&bitmap, &treemap); + assert_eq!( + bitmap.cardinality(), + lhs_bitmap.andnot_cardinality(rhs_bitmap), + ); + } + } + } +} + +pub fn assert_64_eq(lhs: &Bitmap64, rhs: &Treemap) { + assert_eq!(lhs.cardinality(), rhs.cardinality()); + let lhs_ser = lhs.serialize::(); + let rhs_ser = rhs.serialize::(); + if lhs_ser != rhs_ser { + let mut lhs_it = lhs.iter().enumerate(); + let mut rhs_it = rhs.iter(); + while let Some((i, l)) = lhs_it.next() { + let r = rhs_it.next().unwrap(); + assert_eq!(l, r, "{l} != {r} at {i}"); + } + assert_eq!(rhs_it.next(), None); + } +} diff --git a/fuzz/fuzz_targets/fuzz_ops64.rs b/fuzz/fuzz_targets/fuzz_ops64.rs new file mode 100644 index 0000000..aad846a --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_ops64.rs @@ -0,0 +1,54 @@ +#![no_main] + +use crate::arbitrary_ops64::*; +use croaring::{Bitmap64, Portable, Treemap}; +use libfuzzer_sys::arbitrary; +use libfuzzer_sys::arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; +use std::collections::HashSet; + +mod arbitrary_ops64; + +fuzz_target!(|input: FuzzInput| { + let mut lhs64 = Bitmap64::deserialize::(input.initial_input); + let mut rhs64 = Bitmap64::new(); + let mut lhs_tree = Treemap::from_iter(lhs64.iter()); + let mut rhs_tree = Treemap::new(); + + for op in input.lhs_ops.iter() { + op.on_bitmap64(&mut lhs64); + lhs64.internal_validate().unwrap(); + op.on_treemap(&mut lhs_tree); + } + for op in input.rhs_ops.iter() { + op.on_bitmap64(&mut rhs64, &lhs64); + rhs64.internal_validate().unwrap(); + op.on_treemap(&mut rhs_tree, &lhs_tree); + } + for op in input.compares.iter() { + op.compare_with_tree(&lhs64, &rhs64, &lhs_tree, &rhs_tree); + } + for op in input.view_ops.iter() { + op.check_against_tree(&lhs64, &lhs_tree); + } + + { + let mut cursor = lhs64.cursor(); + for op in input.iter_ops.iter() { + op.on_cursor(&lhs64, &mut cursor); + } + } + + assert_64_eq(&lhs64, &lhs_tree); + assert_64_eq(&rhs64, &rhs_tree); +}); + +#[derive(Arbitrary, Debug)] +struct FuzzInput<'a> { + lhs_ops: Vec, + rhs_ops: Vec, + compares: HashSet, + view_ops: HashSet, + iter_ops: Vec, + initial_input: &'a [u8], +}