[fix][offload] Increase file upload limit from 2048MiB to 4096MiB for…

… GCP/GCS offloading (apache#22554)

conf/broker.conf

@@ -1670,10 +1670,10 @@ s3ManagedLedgerOffloadBucket=
 # For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing)
 s3ManagedLedgerOffloadServiceEndpoint=
 
-# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
+# For Amazon S3 ledger offload, Max block size in bytes. (64MiB by default, 5MiB minimum)
 s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
 
-# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
+# For Amazon S3 ledger offload, Read buffer size in bytes (1MiB by default)
 s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage ledger offload, region where offload bucket is located.
@@ -1683,10 +1683,11 @@ gcsManagedLedgerOffloadRegion=
 # For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into
 gcsManagedLedgerOffloadBucket=
 
-# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
-gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864
+# For Google Cloud Storage ledger offload, Max block size in bytes. (128MiB by default, 5MiB minimum)
+# Since JClouds limits the maximum number of blocks to 32, the maximum size of a ledger is 32 times the block size.
+gcsManagedLedgerOffloadMaxBlockSizeInBytes=134217728
 
-# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default)
+# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MiB by default)
 gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576
 
 # For Google Cloud Storage, path to json file containing service account credentials.

pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java

@@ -79,8 +79,9 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies {
         }
     }
 
-    public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024;   // 64MB
-    public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024;      // 1MB
+    public static final int DEFAULT_MAX_BLOCK_SIZE_IN_BYTES = 64 * 1024 * 1024;   // 64MiB
+    public static final int DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES = 128 * 1024 * 1024;   // 128MiB
+    public static final int DEFAULT_READ_BUFFER_SIZE_IN_BYTES = 1024 * 1024;      // 1MiB
     public static final int DEFAULT_OFFLOAD_MAX_THREADS = 2;
     public static final int DEFAULT_OFFLOAD_MAX_PREFETCH_ROUNDS = 1;
     public static final String DEFAULT_OFFLOADER_DIRECTORY = "./offloaders";
@@ -163,7 +164,7 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies {
     private String gcsManagedLedgerOffloadBucket = null;
     @Configuration
     @JsonProperty(access = JsonProperty.Access.READ_WRITE)
-    private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_MAX_BLOCK_SIZE_IN_BYTES;
+    private Integer gcsManagedLedgerOffloadMaxBlockSizeInBytes = DEFAULT_GCS_MAX_BLOCK_SIZE_IN_BYTES;
     @Configuration
     @JsonProperty(access = JsonProperty.Access.READ_WRITE)
     private Integer gcsManagedLedgerOffloadReadBufferSizeInBytes = DEFAULT_READ_BUFFER_SIZE_IN_BYTES;