diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0e899e3e7051e..584eab6527a65 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -152,6 +152,7 @@ public class S3AFileSystem extends FileSystem { * for this FileSystem * @param conf the configuration */ + @SuppressWarnings("deprecation") public void initialize(URI name, Configuration conf) throws IOException { super.initialize(name, conf); setConf(conf); @@ -181,7 +182,7 @@ public void initialize(URI name, Configuration conf) throws IOException { longOption(conf, FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE, 1); enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true); - readAhead = longOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0); + readAhead = longByteOption(conf, READAHEAD_RANGE, DEFAULT_READAHEAD_RANGE, 0); storageStatistics = (S3AStorageStatistics) GlobalStorageStatistics.INSTANCE .put(S3AStorageStatistics.NAME, @@ -1868,10 +1869,11 @@ private ObjectMetadata cloneObjectMetadata(ObjectMetadata source) { */ @Deprecated public long getDefaultBlockSize() { - return getConf().getLong(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE); + return getConf().getLongBytes(FS_S3A_BLOCK_SIZE, DEFAULT_BLOCKSIZE); } @Override + @SuppressWarnings("deprecation") public String toString() { final StringBuilder sb = new StringBuilder( "S3AFileSystem{"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index c89f6904cdaa6..22aa10ca74584 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -462,6 +462,27 @@ static long longOption(Configuration conf, return v; } + /** + * Get a long option >= the minimum allowed value, supporting memory + * prefixes K,M,G,T,P. + * @param conf configuration + * @param key key to look up + * @param defVal default value + * @param min minimum value + * @return the value + * @throws IllegalArgumentException if the value is below the minimum + */ + static long longByteOption(Configuration conf, + String key, + long defVal, + long min) { + long v = conf.getLongBytes(key, defVal); + Preconditions.checkArgument(v >= min, + String.format("Value of %s: %d is below the minimum value %d", + key, v, min)); + return v; + } + /** * Get a size property from the configuration: this property must * be at least equal to {@link Constants#MULTIPART_MIN_SIZE}. @@ -474,7 +495,7 @@ static long longOption(Configuration conf, */ public static long getMultipartSizeProperty(Configuration conf, String property, long defVal) { - long partSize = conf.getLong(property, defVal); + long partSize = conf.getLongBytes(property, defVal); if (partSize < MULTIPART_MIN_SIZE) { LOG.warn("{} must be at least 5 MB; configured value is {}", property, partSize);