From 71d9047da6e229019ecad747c2d4e61ee5ee39b6 Mon Sep 17 00:00:00 2001
From: A-Stupid-Sun <jimmyholoveslife@foxmail.com>
Date: Mon, 5 Jun 2023 19:05:15 +0800
Subject: [PATCH] fix(services/s3): Fix s3 batch max operations

---
 core/src/services/s3/backend.rs | 17 ++++++++++++++---
 core/src/services/s3/core.rs    |  1 +
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
index 99ec8bfab904..be720a8f9614 100644
--- a/core/src/services/s3/backend.rs
+++ b/core/src/services/s3/backend.rs
@@ -55,7 +55,7 @@ static ENDPOINT_TEMPLATES: Lazy<HashMap<&'static str, &'static str>> = Lazy::new
 });
 
 const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024;
-
+const DEFAULT_BATCH_MAX_OPERATIONS: usize = 1000;
 /// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support.
 /// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services).
 ///
@@ -93,6 +93,8 @@ pub struct S3Builder {
     /// the part size of s3 multipart upload, which should be 5 MiB to 5 GiB.
     /// There is no minimum size limit on the last part of your multipart upload
     write_min_size: Option<usize>,
+    /// batch_max_operations
+    batch_max_operations: Option<usize>,
 }
 
 impl Debug for S3Builder {
@@ -508,6 +510,12 @@ impl S3Builder {
     pub fn write_min_size(&mut self, write_min_size: usize) -> &mut Self {
         self.write_min_size = Some(write_min_size);
 
+        self
+    }
+    /// Set maximum batch operations of this backend.
+    pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self {
+        self.batch_max_operations = Some(batch_max_operations);
+
         self
     }
 }
@@ -696,7 +704,9 @@ impl Builder for S3Builder {
             )
             .with_context("service", Scheme::S3));
         }
-
+        let batch_max_operations = self
+            .batch_max_operations
+            .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
         debug!("backend build finished");
         Ok(S3Backend {
             core: Arc::new(S3Core {
@@ -714,6 +724,7 @@ impl Builder for S3Builder {
                 loader,
                 client,
                 write_min_size,
+                batch_max_operations,
             }),
         })
     }
@@ -773,7 +784,7 @@ impl Accessor for S3Backend {
                 presign_write: true,
 
                 batch: true,
-                batch_max_operations: Some(1000),
+                batch_max_operations: Some(self.core.batch_max_operations),
 
                 ..Default::default()
             });
diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs
index bc38fe194b57..c802fd430283 100644
--- a/core/src/services/s3/core.rs
+++ b/core/src/services/s3/core.rs
@@ -82,6 +82,7 @@ pub struct S3Core {
     pub loader: AwsLoader,
     pub client: HttpClient,
     pub write_min_size: usize,
+    pub batch_max_operations: usize,
 }
 
 impl Debug for S3Core {