diff --git a/bindings/java/src/async_operator.rs b/bindings/java/src/async_operator.rs index 06231d7906e0..7835b2256684 100644 --- a/bindings/java/src/async_operator.rs +++ b/bindings/java/src/async_operator.rs @@ -34,9 +34,9 @@ use opendal::Operator; use opendal::Scheme; use crate::convert::{ - bytes_to_jbytearray, jmap_to_hashmap, offset_length_to_range, read_int64_field, + bytes_to_jbytearray, jmap_to_hashmap, jstring_to_string, offset_length_to_range, + read_int64_field, }; -use crate::convert::{jstring_to_string, read_bool_field}; use crate::executor::executor_or_default; use crate::executor::get_current_env; use crate::executor::Executor; @@ -44,7 +44,7 @@ use crate::make_metadata; use crate::make_operator_info; use crate::make_presigned_request; use crate::Result; -use crate::{make_entry, make_write_options}; +use crate::{make_entry, make_list_options, make_write_options}; #[no_mangle] pub extern "system" fn Java_org_apache_opendal_AsyncOperator_constructor( @@ -504,13 +504,9 @@ fn intern_list( let id = request_id(env)?; let path = jstring_to_string(env, &path)?; - let recursive = read_bool_field(env, &options, "recursive")?; - - let mut list_op = op.list_with(&path); - list_op = list_op.recursive(recursive); - + let list_opts = make_list_options(env, &options)?; executor_or_default(env, executor)?.spawn(async move { - let entries = list_op.await.map_err(Into::into); + let entries = op.list_options(&path, list_opts).await.map_err(Into::into); let result = make_entries(entries); complete_future(id, result.map(JValueOwned::Object)) }); diff --git a/bindings/java/src/convert.rs b/bindings/java/src/convert.rs index f163722cc186..78e2497622fc 100644 --- a/bindings/java/src/convert.rs +++ b/bindings/java/src/convert.rs @@ -106,6 +106,22 @@ pub(crate) fn read_map_field( } } +pub(crate) fn read_jlong_field_to_usize( + env: &mut JNIEnv, + options: &JObject, + field_name: &str, +) -> Result> { + match read_int64_field(env, options, field_name)? { + -1 => Ok(None), + v if v > 0 => Ok(Some(v as usize)), + v => Err(Error::new( + ErrorKind::Unexpected, + format!("{} must be positive, instead got: {}", field_name, v), + ) + .into()), + } +} + pub(crate) fn offset_length_to_range(offset: i64, length: i64) -> Result<(Bound, Bound)> { let offset = u64::try_from(offset) .map_err(|_| Error::new(ErrorKind::RangeNotSatisfied, "offset must be non-negative"))?; diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index a2ab50b633ec..21c3359b8b0a 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -94,7 +94,7 @@ fn make_operator_info<'a>(env: &mut JNIEnv<'a>, info: OperatorInfo) -> Result(env: &mut JNIEnv<'a>, cap: Capability) -> Result> { let capability = env.new_object( "org/apache/opendal/Capability", - "(ZZZZZZZZZZZZZZZZZZZJJZZZZZZZZZZZZZ)V", + "(ZZZZZZZZZZZZZZZZZZZJJZZZZZZZZZZZZZZZ)V", &[ JValue::Bool(cap.stat as jboolean), JValue::Bool(cap.stat_with_if_match as jboolean), @@ -125,6 +125,8 @@ fn make_capability<'a>(env: &mut JNIEnv<'a>, cap: Capability) -> Result( .into()) } }; - let chunk = match convert::read_int64_field(env, options, "chunk")? { - -1 => None, - v if v >= 0 => Some(v as usize), - v => { - return Err(Error::new( - ErrorKind::Unexpected, - format!("Chunk must be positive, instead got: {}", v), - ) - .into()) - } - }; - Ok(opendal::options::WriteOptions { append: convert::read_bool_field(env, options, "append").unwrap_or_default(), content_type: convert::read_string_field(env, options, "contentType")?, @@ -240,6 +230,19 @@ fn make_write_options<'a>( if_not_exists: convert::read_bool_field(env, options, "ifNotExists").unwrap_or_default(), user_metadata: convert::read_map_field(env, options, "userMetadata")?, concurrent, - chunk, + chunk: convert::read_jlong_field_to_usize(env, options, "chunk")?, + }) +} + +fn make_list_options<'a>( + env: &mut JNIEnv<'a>, + options: &JObject, +) -> Result { + Ok(opendal::options::ListOptions { + limit: convert::read_jlong_field_to_usize(env, options, "limit")?, + start_after: convert::read_string_field(env, options, "startAfter")?, + recursive: convert::read_bool_field(env, options, "recursive").unwrap_or_default(), + versions: convert::read_bool_field(env, options, "versions").unwrap_or_default(), + deleted: convert::read_bool_field(env, options, "deleted").unwrap_or_default(), }) } diff --git a/bindings/java/src/main/java/org/apache/opendal/Capability.java b/bindings/java/src/main/java/org/apache/opendal/Capability.java index 0e3ad0420f93..d108990c20ab 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Capability.java +++ b/bindings/java/src/main/java/org/apache/opendal/Capability.java @@ -172,6 +172,16 @@ public class Capability { */ public final boolean listWithRecursive; + /** + * If backend support list with versions. + */ + public final boolean listWithVersions; + + /** + * If backend support list with deleted. + */ + public final boolean listWithDeleted; + /** * If operator supports presign. */ @@ -227,6 +237,8 @@ public Capability( boolean listWithLimit, boolean listWithStartAfter, boolean listWithRecursive, + boolean listWithVersions, + boolean listWithDeleted, boolean presign, boolean presignRead, boolean presignStat, @@ -261,6 +273,8 @@ public Capability( this.listWithLimit = listWithLimit; this.listWithStartAfter = listWithStartAfter; this.listWithRecursive = listWithRecursive; + this.listWithVersions = listWithVersions; + this.listWithDeleted = listWithDeleted; this.presign = presign; this.presignRead = presignRead; this.presignStat = presignStat; diff --git a/bindings/java/src/main/java/org/apache/opendal/ListOptions.java b/bindings/java/src/main/java/org/apache/opendal/ListOptions.java index 08356acecea3..9e36752f4587 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ListOptions.java +++ b/bindings/java/src/main/java/org/apache/opendal/ListOptions.java @@ -28,4 +28,27 @@ public class ListOptions { * Return files in subdirectory as well. */ public final boolean recursive; + + /** + * The limit passed to underlying service to specify the max results + * that could return per-request. + */ + @Builder.Default + public final long limit = -1; + + /** + * The startAfter option passes to underlying service to specify the + * specified key to start listing from. + */ + public final String startAfter; + + /** + * The versions option is used to control whether the object versions should be returned. + */ + public final boolean versions; + + /** + * The deleted is used to control whether the deleted objects should be returned. + */ + public final boolean deleted; } diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs index 85bdfef097c8..160ac15ce8de 100644 --- a/bindings/java/src/operator.rs +++ b/bindings/java/src/operator.rs @@ -29,12 +29,11 @@ use opendal::blocking; use opendal::options; use crate::convert::{ - bytes_to_jbytearray, jstring_to_string, offset_length_to_range, read_bool_field, - read_int64_field, + bytes_to_jbytearray, jstring_to_string, offset_length_to_range, read_int64_field, }; use crate::make_metadata; use crate::Result; -use crate::{make_entry, make_write_options}; +use crate::{make_entry, make_list_options, make_write_options}; /// # Safety /// @@ -296,15 +295,8 @@ fn intern_list( options: JObject, ) -> Result { let path = jstring_to_string(env, &path)?; - let recursive = read_bool_field(env, &options, "recursive")?; - - let entries = op.list_options( - &path, - options::ListOptions { - recursive, - ..Default::default() - }, - )?; + let list_opts = make_list_options(env, &options)?; + let entries = op.list_options(&path, list_opts)?; let jarray = env.new_object_array( entries.len() as jsize, diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java index cb3533a49401..f30fa82356e0 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/AsyncListTest.java @@ -251,4 +251,83 @@ public void testListRecursive() { .join(); assertThat(noRecursiveEntries).hasSize(3); } + + @Test + void testListWithLimitCollectsAllPages() { + assumeTrue(asyncOp().info.fullCapability.listWithLimit); + + String dir = String.format("%s/", UUID.randomUUID()); + asyncOp().createDir(dir).join(); + for (int i = 0; i < 5; i++) { + String file = dir + "file-" + i; + asyncOp().write(file, "data").join(); + } + + ListOptions options = ListOptions.builder().limit(3).build(); + List entries = asyncOp().list(dir, options).join(); + assertThat(entries.size()).isEqualTo(6); + + asyncOp().removeAll(dir).join(); + } + + @Test + void testListWithStartAfter() { + assumeTrue(asyncOp().info.fullCapability.listWithStartAfter); + + String dir = String.format("%s/", UUID.randomUUID()); + asyncOp().createDir(dir).join(); + + List filesToCreate = Lists.newArrayList(); + for (int i = 0; i < 5; i++) { + String file = dir + "file-" + i; + asyncOp().write(file, "data").join(); + filesToCreate.add(file); + } + + ListOptions options = + ListOptions.builder().startAfter(filesToCreate.get(2)).build(); + List entries = asyncOp().list(dir, options).join(); + List actual = entries.stream().map(Entry::getPath).sorted().collect(Collectors.toList()); + + assertThat(actual).containsAnyElementsOf(filesToCreate.subList(3, 5)); + asyncOp().removeAll(dir).join(); + } + + @Test + void testListWithVersions() { + assumeTrue(asyncOp().info.fullCapability.listWithVersions); + + String dir = String.format("%s/", UUID.randomUUID()); + String path = dir + "versioned-file"; + + asyncOp().createDir(dir).join(); + asyncOp().write(path, "data-1").join(); + asyncOp().write(path, "data-2").join(); + + ListOptions options = ListOptions.builder().versions(true).build(); + List entries = asyncOp().list(dir, options).join(); + + assertThat(entries).isNotEmpty(); + asyncOp().removeAll(dir).join(); + } + + @Test + void testListWithDeleted() { + assumeTrue(asyncOp().info.fullCapability.listWithDeleted); + + String dir = String.format("%s/", UUID.randomUUID()); + String path = dir + "file"; + asyncOp().createDir(dir).join(); + + asyncOp().write(path, "data").join(); + asyncOp().delete(path).join(); + + ListOptions options = ListOptions.builder().deleted(true).build(); + List entries = asyncOp().list(dir, options).join(); + + assertThat(entries.size()).isEqualTo(1); + assertThat(entries.get(0).getPath()).isEqualTo(path); + + asyncOp().removeAll(dir).join(); + } } diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java index cbb00ea555cc..3f0459af764a 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingListTest.java @@ -25,12 +25,14 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.opendal.Capability; import org.apache.opendal.Entry; import org.apache.opendal.ListOptions; import org.apache.opendal.Metadata; import org.apache.opendal.OpenDALException; import org.apache.opendal.test.condition.OpenDALExceptionCondition; +import org.assertj.core.util.Lists; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -129,4 +131,83 @@ public void testListRecursive() { op().list(dir, ListOptions.builder().recursive(false).build()); assertThat(noRecursiveEntries).hasSize(3); } + + @Test + void testListWithLimitCollectsAllPages() { + assumeTrue(op().info.fullCapability.listWithLimit); + + String dir = String.format("%s/", UUID.randomUUID()); + op().createDir(dir); + for (int i = 0; i < 5; i++) { + String file = dir + "file-" + i; + op().write(file, "data"); + } + + ListOptions options = ListOptions.builder().limit(3).build(); + List entries = op().list(dir, options); + assertThat(entries.size()).isEqualTo(6); + + op().removeAll(dir); + } + + @Test + void testListWithStartAfter() { + assumeTrue(op().info.fullCapability.listWithStartAfter); + + String dir = String.format("%s/", UUID.randomUUID()); + op().createDir(dir); + + List filesToCreate = Lists.newArrayList(); + for (int i = 0; i < 5; i++) { + String file = dir + "file-" + i; + op().write(file, "data"); + filesToCreate.add(file); + } + + ListOptions options = + ListOptions.builder().startAfter(filesToCreate.get(2)).build(); + List entries = op().list(dir, options); + List actual = entries.stream().map(Entry::getPath).sorted().collect(Collectors.toList()); + + assertThat(actual).containsAnyElementsOf(filesToCreate.subList(3, 5)); + op().removeAll(dir); + } + + @Test + void testListWithVersions() { + assumeTrue(op().info.fullCapability.listWithVersions); + + String dir = String.format("%s/", UUID.randomUUID()); + String path = dir + "versioned-file"; + + op().createDir(dir); + op().write(path, "data-1"); + op().write(path, "data-2"); + + ListOptions options = ListOptions.builder().versions(true).build(); + List entries = op().list(dir, options); + + assertThat(entries).isNotEmpty(); + op().removeAll(dir); + } + + @Test + void testListWithDeleted() { + assumeTrue(op().info.fullCapability.listWithDeleted); + + String dir = String.format("%s/", UUID.randomUUID()); + String path = dir + "file"; + + op().createDir(dir); + op().write(path, "data"); + op().delete(path); + + ListOptions options = ListOptions.builder().deleted(true).build(); + List entries = op().list(dir, options); + + assertThat(entries.size()).isEqualTo(1); + assertThat(entries.get(0).getPath()).isEqualTo(path); + + op().removeAll(dir); + } }